From 3c822489400757bb030fc955a10c97582dd7252f Mon Sep 17 00:00:00 2001 From: "claude[bot]" <209825114+claude[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 10:59:16 +0100 Subject: [PATCH 01/67] chore(deps): bump tar to 7.5.19 (#4345) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pins `tar` to `7.5.19` via a root `pnpm.overrides` entry, replacing a stale range override (`tar@>=7 <7.5.11`) that no longer matched any installed copy. The single override collapses all resolved `tar` copies onto one version: - `packages/cli-v3` — direct dependency (was 7.5.13) - `@kubernetes/client-node` (apps/supervisor) — transitive (was 7.5.13) - `cacache` — transitive (was 6.2.1) - `giget` — transitive (was 6.2.1) No source changes; cli-v3's published `^7.5.13` spec already permits `7.5.19`, so no changeset is needed. --- package.json | 2 +- pnpm-lock.yaml | 84 +++++++++----------------------------------------- 2 files changed, 15 insertions(+), 71 deletions(-) diff --git a/package.json b/package.json index dddb2275380..3b00102ac83 100644 --- a/package.json +++ b/package.json @@ -102,7 +102,7 @@ "@ai-sdk/provider-utils@^4": "4.0.29", "express@^4>body-parser": "1.20.3", "@remix-run/dev@2.17.5>tar-fs": "2.1.4", - "tar@>=7 <7.5.11": "^7.5.11", + "tar": "7.5.19", "form-data@^2": "2.5.4", "form-data@^3": "3.0.5", "form-data@^4": "4.0.6", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cd2d2d93fba..b09dc718a6f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,7 +13,7 @@ overrides: '@ai-sdk/provider-utils@^4': 4.0.29 express@^4>body-parser: 1.20.3 '@remix-run/dev@2.17.5>tar-fs': 2.1.4 - tar@>=7 <7.5.11: ^7.5.11 + tar: 7.5.19 form-data@^2: 2.5.4 form-data@^3: 3.0.5 form-data@^4: 4.0.6 @@ -1625,8 +1625,8 @@ importers: specifier: ^10.0.0 version: 10.0.0 tar: - specifier: ^7.5.13 - version: 7.5.13 + specifier: 7.5.19 + version: 7.5.19 tiny-invariant: specifier: ^1.2.0 version: 1.3.1 @@ -3226,19 +3226,19 @@ packages: '@esbuild-kit/core-utils@3.0.0': resolution: {integrity: sha512-TXmwH9EFS3DC2sI2YJWJBgHGhlteK0Xyu1VabwetMULfm3oYhbrsWV5yaSr2NTWZIgDGVLHbRf0inxbjXqAcmQ==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild-kit/core-utils@3.3.2': resolution: {integrity: sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild-kit/esm-loader@2.5.4': resolution: {integrity: sha512-afmtLf6uqxD5IgwCzomtqCYIgz/sjHzCWZFvfS5+FzeYxOURPUo4QcHtqJxbxWOMOogKriZanN/1bJQE/ZL93A==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild-kit/esm-loader@2.6.5': resolution: {integrity: sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild/aix-ppc64@0.23.0': resolution: {integrity: sha512-3sG8Zwa5fMcA9bgqB8AfWPQ+HFke6uD3h1s3RIwUNK8EG7a4buxvuFTs3j1IMs2NXAk9F30C/FF4vxRgQCcmoQ==} @@ -8896,10 +8896,6 @@ packages: chownr@1.1.4: resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} - chownr@2.0.0: - resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==} - engines: {node: '>=10'} - chownr@3.0.0: resolution: {integrity: sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==} engines: {node: '>=18'} @@ -10576,10 +10572,6 @@ packages: resolution: {integrity: sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==} engines: {node: '>=6 <7 || >=8'} - fs-minipass@2.1.0: - resolution: {integrity: sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==} - engines: {node: '>= 8'} - fs-minipass@3.0.3: resolution: {integrity: sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -12212,22 +12204,10 @@ packages: resolution: {integrity: sha512-g2Uuh2jEKoht+zvO6vJqXmYpflPqzRBT+Th2h01DKh5z7wbY/AZ2gCQ78cP70YoHPyFdY30YBV5WxgLOEwOykw==} engines: {node: '>=8'} - minipass@5.0.0: - resolution: {integrity: sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==} - engines: {node: '>=8'} - - minipass@7.1.2: - resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} - engines: {node: '>=16 || 14 >=14.17'} - minipass@7.1.3: resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==} engines: {node: '>=16 || 14 >=14.17'} - minizlib@2.1.2: - resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==} - engines: {node: '>= 8'} - minizlib@3.1.0: resolution: {integrity: sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==} engines: {node: '>= 18'} @@ -12239,11 +12219,6 @@ packages: mkdirp-classic@0.5.3: resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} - mkdirp@1.0.4: - resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} - engines: {node: '>=10'} - hasBin: true - mkdirp@3.0.1: resolution: {integrity: sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==} engines: {node: '>=10'} @@ -14526,13 +14501,8 @@ packages: tar-stream@3.1.7: resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} - tar@6.2.1: - resolution: {integrity: sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==} - engines: {node: '>=10'} - deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me - - tar@7.5.13: - resolution: {integrity: sha512-tOG/7GyXpFevhXVh8jOPJrmtRpOTsYqUIkVdVooZYJS/z8WhfQUX8RJILmeuJNinGAMSu1veBr4asSHFt5/hng==} + tar@7.5.19: + resolution: {integrity: sha512-4LeEWl96twnS2Q7Bz4MGqgazLqO+hJN63GZxXoIqh1T3VweYD997gbU1ItNsQafqqXTXd5WFyFdReLtwvRBNiw==} engines: {node: '>=18'} tdigest@0.1.2: @@ -18795,7 +18765,7 @@ snapshots: openid-client: 6.3.3 rfc4648: 1.5.3 stream-buffers: 3.0.2 - tar: 7.5.13 + tar: 7.5.19 tmp-promise: 3.0.3 tslib: 2.6.2 ws: 8.21.0(bufferutil@4.0.9) @@ -23713,7 +23683,7 @@ snapshots: minipass-pipeline: 1.2.4 p-map: 4.0.0 ssri: 10.0.5 - tar: 6.2.1 + tar: 7.5.19 unique-filename: 3.0.0 cacheable-request@6.1.0: @@ -23823,8 +23793,6 @@ snapshots: chownr@1.1.4: {} - chownr@2.0.0: {} - chownr@3.0.0: {} chrome-trace-event@1.0.4: {} @@ -25801,10 +25769,6 @@ snapshots: jsonfile: 4.0.0 universalify: 0.1.2 - fs-minipass@2.1.0: - dependencies: - minipass: 3.3.6 - fs-minipass@3.0.3: dependencies: minipass: 7.1.3 @@ -25910,7 +25874,7 @@ snapshots: nypm: 0.3.9 ohash: 1.1.3 pathe: 1.1.2 - tar: 6.2.1 + tar: 7.5.19 giget@2.0.0: dependencies: @@ -27832,17 +27796,8 @@ snapshots: dependencies: yallist: 4.0.0 - minipass@5.0.0: {} - - minipass@7.1.2: {} - minipass@7.1.3: {} - minizlib@2.1.2: - dependencies: - minipass: 3.3.6 - yallist: 4.0.0 - minizlib@3.1.0: dependencies: minipass: 7.1.3 @@ -27851,8 +27806,6 @@ snapshots: mkdirp-classic@0.5.3: {} - mkdirp@1.0.4: {} - mkdirp@3.0.1: {} ml-array-max@1.2.4: @@ -30524,20 +30477,11 @@ snapshots: transitivePeerDependencies: - bare-abort-controller - tar@6.2.1: - dependencies: - chownr: 2.0.0 - fs-minipass: 2.1.0 - minipass: 5.0.0 - minizlib: 2.1.2 - mkdirp: 1.0.4 - yallist: 4.0.0 - - tar@7.5.13: + tar@7.5.19: dependencies: '@isaacs/fs-minipass': 4.0.1 chownr: 3.0.0 - minipass: 7.1.2 + minipass: 7.1.3 minizlib: 3.1.0 yallist: 5.0.0 From 88ca0091a9c53fc1e220bcb34cf52ed471a6bcb7 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Thu, 23 Jul 2026 11:51:34 +0100 Subject: [PATCH 02/67] fix(docker): stop the container entrypoint printing database connection strings in logs (#4346) ## Summary The container entrypoint runs under `set -x`, which echoes every command to the logs with its variables expanded. Several startup guards reference full database connection strings, so the DSN (including the password) was printed to the container logs on every boot. This turns tracing off around those lines so connection strings are never traced, while leaving migration behavior and ordinary startup logging unchanged. ## Fix The leaking lines are the `[ -n "$RUN_OPS_DATABASE_URL" ]` and `[ -n "$RUN_OPS_LEGACY_DIRECT_URL" ]` guards, and the ClickHouse block (its `[ -n "$CLICKHOUSE_URL" ]` guard plus the lines that build `GOOSE_DBSTRING` from `CLICKHOUSE_URL`). `set -x` prints each of these with the credential expanded. Tracing is now disabled around each region and restored afterward, so non-secret tracing is preserved everywhere else. The existing legacy-migration subshell already protected its own command body; this adds the missing protection for the guards and the ClickHouse block. ```sh { set +x; } 2>/dev/null if [ -n "$RUN_OPS_DATABASE_URL" ]; then set -x ... ``` ## Verification Built the webapp image and ran it with dummy sentinel connection strings whose password token is `S3NTINEL_PW_DoNotLog`, then grepped the boot logs. Before (unmodified), the token appears in the traced guards: ``` + [ -n postgresql://user:S3NTINEL_PW_DoNotLog@fake-host:6432/run-ops ] + [ -n postgresql://user:S3NTINEL_PW_DoNotLog@fake-host:5432/legacy ] + [ -n https://default:S3NTINEL_PW_DoNotLog@fake-host:8443 ] ``` After, `grep S3NTINEL_PW_DoNotLog` on the same run returns nothing, and the normal "skipping ... migrations" lines still log. --- .../entrypoint-no-log-db-connection-strings.md | 6 ++++++ docker/scripts/entrypoint.sh | 8 ++++++++ 2 files changed, 14 insertions(+) create mode 100644 .server-changes/entrypoint-no-log-db-connection-strings.md diff --git a/.server-changes/entrypoint-no-log-db-connection-strings.md b/.server-changes/entrypoint-no-log-db-connection-strings.md new file mode 100644 index 00000000000..611408cf14c --- /dev/null +++ b/.server-changes/entrypoint-no-log-db-connection-strings.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Container startup no longer prints database and ClickHouse connection strings (with credentials) to the logs. diff --git a/docker/scripts/entrypoint.sh b/docker/scripts/entrypoint.sh index 1871755f985..1e5c7c7cab0 100755 --- a/docker/scripts/entrypoint.sh +++ b/docker/scripts/entrypoint.sh @@ -15,7 +15,9 @@ fi # Run-ops split: migrate the dedicated NEW run-ops database only when it is configured. Single-DB # installs never set the URL, so this is a no-op there. +{ set +x; } 2>/dev/null if [ -n "$RUN_OPS_DATABASE_URL" ]; then + set -x if [ "$SKIP_RUN_OPS_MIGRATIONS" != "1" ]; then echo "Running run-ops migrations" pnpm --filter @internal/run-ops-database db:migrate:deploy @@ -24,13 +26,16 @@ if [ -n "$RUN_OPS_DATABASE_URL" ]; then echo "SKIP_RUN_OPS_MIGRATIONS=1, skipping run-ops migrations." fi else + set -x echo "RUN_OPS_DATABASE_URL not set, skipping run-ops migrations." fi # Run-ops split: keep the legacy runs DB's schema current by applying the full @trigger.dev/database # migrations to it too, pointed at its direct (non-pooled) URL. Only runs when that URL is configured; # installs that never set it skip this entirely. +{ set +x; } 2>/dev/null if [ -n "$RUN_OPS_LEGACY_DIRECT_URL" ]; then + set -x if [ "$SKIP_RUN_OPS_LEGACY_MIGRATIONS" != "1" ]; then echo "Running legacy run-ops migrations" # Subshell with tracing off so `set -x` does not print the DSN (with credentials) to the logs. @@ -40,6 +45,7 @@ if [ -n "$RUN_OPS_LEGACY_DIRECT_URL" ]; then echo "SKIP_RUN_OPS_LEGACY_MIGRATIONS=1, skipping legacy run-ops migrations." fi else + set -x echo "RUN_OPS_LEGACY_DIRECT_URL not set, skipping legacy run-ops migrations." fi @@ -51,6 +57,7 @@ else echo "SKIP_DASHBOARD_AGENT_MIGRATIONS=1, skipping dashboard agent migrations." fi +{ set +x; } 2>/dev/null if [ -n "$CLICKHOUSE_URL" ] && [ "$SKIP_CLICKHOUSE_MIGRATIONS" != "1" ]; then # Run ClickHouse migrations echo "Running ClickHouse migrations..." @@ -76,6 +83,7 @@ elif [ "$SKIP_CLICKHOUSE_MIGRATIONS" = "1" ]; then else echo "CLICKHOUSE_URL not set, skipping ClickHouse migrations." fi +set -x # Copy over required prisma files cp internal-packages/database/prisma/schema.prisma apps/webapp/prisma/ From 722e240e4d74a72fd072bd7f6e1f2adbb6a3f36e Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:18:58 +0100 Subject: [PATCH 03/67] feat(supervisor): add prometheus metric for outbound http requests (#4350) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Prometheus metrics so the supervisor's outbound HTTP calls are observable - including client-side failures that previously only surfaced as a log line. - `supervisor_outbound_request_total{name, method, status, outcome}` - counts every outbound request. `outcome` separates a transport failure (`network_error`), an HTTP error response (`http_error`), a response that failed schema validation (`invalid_response`), and success (`ok`). - `supervisor_outbound_request_duration_seconds{name, outcome}` - latency histogram. Leaner labels than the counter (no `status`) to avoid bucket×label cardinality; buckets match the existing dequeue-latency histogram since these calls share the same retrying HTTP client and long-poll envelope. Coverage: - The warm-start request (a one-off `fetch`) - instrumented inline; the response status code is now also included in the failure log (it was previously dropped). - All worker API client calls (`SupervisorHttpClient`: dequeue, run attempt start/complete, heartbeats, snapshots, continue, suspend, debug-log, connect) - routed through a single instrumented `request()` helper that reports via an optional `onHttpRequestComplete` callback on the client, which the supervisor wires into the counter + histogram. Low cardinality by design: `name` is a **static per-endpoint label** (e.g. `dequeue`, `start_run_attempt`), never the interpolated URL - so no run/snapshot IDs land in labels, mirroring the templated `route` labels on the inbound HTTP server. Registered on the existing metrics registry, exposed on `/metrics` with no new wiring. Internal-only change (no package release needed), so the changelog note is a single `.server-changes` entry. --- .../supervisor-outbound-request-metrics.md | 6 ++ apps/supervisor/src/index.ts | 39 +++++++- .../src/v3/runEngineWorker/supervisor/http.ts | 88 ++++++++++++++++--- .../v3/runEngineWorker/supervisor/types.ts | 9 ++ 4 files changed, 127 insertions(+), 15 deletions(-) create mode 100644 .server-changes/supervisor-outbound-request-metrics.md diff --git a/.server-changes/supervisor-outbound-request-metrics.md b/.server-changes/supervisor-outbound-request-metrics.md new file mode 100644 index 00000000000..c66a499b815 --- /dev/null +++ b/.server-changes/supervisor-outbound-request-metrics.md @@ -0,0 +1,6 @@ +--- +area: supervisor +type: improvement +--- + +Improved supervisor observability: it now reports metrics for its outbound requests, making failed calls to upstream services easier to monitor. diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index cf73be90eea..f30203df547 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -22,7 +22,7 @@ import { isKubernetesEnvironment, } from "@trigger.dev/core/v3/serverOnly"; import { createK8sApi, createApiserverMetricsFetcher } from "./clients/kubernetes.js"; -import { collectDefaultMetrics, Gauge, Histogram } from "prom-client"; +import { collectDefaultMetrics, Counter, Gauge, Histogram } from "prom-client"; import { register } from "./metrics.js"; import { PodCleaner } from "./services/podCleaner.js"; import { FailedPodHandler } from "./services/failedPodHandler.js"; @@ -60,6 +60,21 @@ const workloadCreateDuration = new Histogram({ registers: [register], }); +const outboundRequestsTotal = new Counter({ + name: "supervisor_outbound_request_total", + help: "Count of outbound HTTP requests from the supervisor, by target name, method, response status, and outcome (ok, http_error, invalid_response, network_error).", + labelNames: ["name", "method", "status", "outcome"], + registers: [register], +}); + +const outboundRequestDuration = new Histogram({ + name: "supervisor_outbound_request_duration_seconds", + help: "Duration of outbound HTTP requests from the supervisor, by target name and outcome. Includes the HTTP client's internal retries and backoff.", + labelNames: ["name", "outcome"], + buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 11, 12.5, 15, 20, 30, 60], + registers: [register], +}); + class ManagedSupervisor { private readonly workerSession: SupervisorSession; private readonly metricsServer?: HttpServer; @@ -322,6 +337,10 @@ class ManagedSupervisor { runNotificationsEnabled: env.TRIGGER_WORKLOAD_API_ENABLED, heartbeatIntervalSeconds: env.TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS, sendRunDebugLogs: env.SEND_RUN_DEBUG_LOGS, + onHttpRequestComplete: ({ name, method, status, outcome, durationMs }) => { + outboundRequestsTotal.inc({ name, method, status, outcome }); + outboundRequestDuration.observe({ name, outcome }, durationMs / 1000); + }, preDequeue: async () => { // Synchronous, hot-path-safe cached read; false when no monitors are active. const skipForBackpressure = this.backpressureMonitors.some((m) => m.shouldSkipDequeue()); @@ -692,6 +711,18 @@ class ManagedSupervisor { headers.traceparent = traceparent; } + const requestStart = performance.now(); + const record = ( + status: string, + outcome: "ok" | "http_error" | "invalid_response" | "network_error" + ) => { + outboundRequestsTotal.inc({ name: "warm_start", method: "POST", status, outcome }); + outboundRequestDuration.observe( + { name: "warm_start", outcome }, + (performance.now() - requestStart) / 1000 + ); + }; + try { const res = await fetch(warmStartUrlWithPath.href, { method: "POST", @@ -700,8 +731,10 @@ class ManagedSupervisor { }); if (!res.ok) { + record(String(res.status), "http_error"); this.logger.error("Warm start failed", { runId: dequeuedMessage.run.id, + statusCode: res.status, }); return false; } @@ -710,6 +743,7 @@ class ManagedSupervisor { const parsedData = z.object({ didWarmStart: z.boolean() }).safeParse(data); if (!parsedData.success) { + record(String(res.status), "invalid_response"); this.logger.error("Warm start response invalid", { runId: dequeuedMessage.run.id, data, @@ -717,8 +751,11 @@ class ManagedSupervisor { return false; } + record(String(res.status), "ok"); + return parsedData.data.didWarmStart; } catch (error) { + record("none", "network_error"); this.logger.error("Warm start error", { runId: dequeuedMessage.run.id, error, diff --git a/packages/core/src/v3/runEngineWorker/supervisor/http.ts b/packages/core/src/v3/runEngineWorker/supervisor/http.ts index d6d51b1c2b2..450be432577 100644 --- a/packages/core/src/v3/runEngineWorker/supervisor/http.ts +++ b/packages/core/src/v3/runEngineWorker/supervisor/http.ts @@ -21,9 +21,9 @@ import { WorkerApiSuspendRunResponseBody, WorkerApiRunSnapshotsSinceResponseBody, } from "./schemas.js"; -import type { SupervisorClientCommonOptions } from "./types.js"; +import type { SupervisorClientCommonOptions, SupervisorHttpRequestMetric } from "./types.js"; import { getDefaultWorkerHeaders } from "./util.js"; -import { wrapZodFetch } from "../../zodfetch.js"; +import { wrapZodFetch, type ApiResult, type ZodFetchOptions } from "../../zodfetch.js"; import { createHeaders } from "../util.js"; import { WORKER_HEADERS } from "../consts.js"; import { SimpleStructuredLogger } from "../../utils/structuredLogger.js"; @@ -36,6 +36,7 @@ export class SupervisorHttpClient { private readonly instanceName: string; private readonly defaultHeaders: Record; private readonly sendRunDebugLogs: boolean; + private readonly onHttpRequestComplete?: (metric: SupervisorHttpRequestMetric) => void; private readonly logger = new SimpleStructuredLogger("supervisor-http-client"); @@ -45,6 +46,7 @@ export class SupervisorHttpClient { this.instanceName = opts.instanceName; this.defaultHeaders = getDefaultWorkerHeaders(opts); this.sendRunDebugLogs = opts.sendRunDebugLogs ?? false; + this.onHttpRequestComplete = opts.onHttpRequestComplete; if (!this.apiUrl) { throw new Error("apiURL is required and needs to be a non-empty string"); @@ -59,8 +61,55 @@ export class SupervisorHttpClient { } } + private async request( + name: string, + schema: T, + url: string, + requestInit?: RequestInit, + options?: ZodFetchOptions> + ): Promise>> { + const start = performance.now(); + const result = await wrapZodFetch(schema, url, requestInit, options); + + if (this.onHttpRequestComplete) { + const durationMs = performance.now() - start; + const method = requestInit?.method ?? "GET"; + + if (result.success) { + this.onHttpRequestComplete({ name, method, status: "2xx", outcome: "ok", durationMs }); + } else if (result.statusCode === 200) { + this.onHttpRequestComplete({ + name, + method, + status: "200", + outcome: "invalid_response", + durationMs, + }); + } else if (typeof result.statusCode === "number") { + this.onHttpRequestComplete({ + name, + method, + status: String(result.statusCode), + outcome: "http_error", + durationMs, + }); + } else { + this.onHttpRequestComplete({ + name, + method, + status: "none", + outcome: "network_error", + durationMs, + }); + } + } + + return result; + } + async connect(body: WorkerApiConnectRequestBody) { - return wrapZodFetch( + return this.request( + "connect", WorkerApiConnectResponseBody, `${this.apiUrl}/engine/v1/worker-actions/connect`, { @@ -75,7 +124,8 @@ export class SupervisorHttpClient { } async dequeue(body: WorkerApiDequeueRequestBody) { - return wrapZodFetch( + return this.request( + "dequeue", WorkerApiDequeueResponseBody, `${this.apiUrl}/engine/v1/worker-actions/dequeue`, { @@ -91,7 +141,8 @@ export class SupervisorHttpClient { /** @deprecated Not currently used */ async dequeueFromVersion(deploymentId: string, maxRunCount = 1, runnerId?: string) { - return wrapZodFetch( + return this.request( + "dequeue_from_version", WorkerApiDequeueResponseBody, `${this.apiUrl}/engine/v1/worker-actions/deployments/${deploymentId}/dequeue?maxRunCount=${maxRunCount}`, { @@ -104,7 +155,8 @@ export class SupervisorHttpClient { } async heartbeatWorker(body: WorkerApiHeartbeatRequestBody) { - return wrapZodFetch( + return this.request( + "heartbeat_worker", WorkerApiHeartbeatResponseBody, `${this.apiUrl}/engine/v1/worker-actions/heartbeat`, { @@ -124,7 +176,8 @@ export class SupervisorHttpClient { body: WorkerApiRunHeartbeatRequestBody, runnerId?: string ) { - return wrapZodFetch( + return this.request( + "heartbeat_run", WorkerApiRunHeartbeatResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/heartbeat`, { @@ -146,7 +199,8 @@ export class SupervisorHttpClient { runnerId?: string, environmentId?: string ) { - return wrapZodFetch( + return this.request( + "start_run_attempt", WorkerApiRunAttemptStartResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/attempts/start`, { @@ -168,7 +222,8 @@ export class SupervisorHttpClient { runnerId?: string, environmentId?: string ) { - return wrapZodFetch( + return this.request( + "complete_run_attempt", WorkerApiRunAttemptCompleteResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/attempts/complete`, { @@ -184,7 +239,8 @@ export class SupervisorHttpClient { } async getLatestSnapshot(runId: string, runnerId?: string, environmentId?: string) { - return wrapZodFetch( + return this.request( + "get_latest_snapshot", WorkerApiRunLatestSnapshotResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/latest`, { @@ -204,7 +260,8 @@ export class SupervisorHttpClient { runnerId?: string, environmentId?: string ) { - return wrapZodFetch( + return this.request( + "get_snapshots_since", WorkerApiRunSnapshotsSinceResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/since/${snapshotId}`, { @@ -224,7 +281,8 @@ export class SupervisorHttpClient { } try { - const res = await wrapZodFetch( + const res = await this.request( + "send_debug_log", z.unknown(), `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/logs/debug`, { @@ -252,7 +310,8 @@ export class SupervisorHttpClient { runnerId?: string, environmentId?: string ) { - return wrapZodFetch( + return this.request( + "continue_run_execution", WorkerApiContinueRunExecutionRequestBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/continue`, { @@ -292,7 +351,8 @@ export class SupervisorHttpClient { runnerId?: string; body: WorkerApiSuspendRunRequestBody; }) { - return wrapZodFetch( + return this.request( + "submit_suspend_completion", WorkerApiSuspendRunResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/suspend`, { diff --git a/packages/core/src/v3/runEngineWorker/supervisor/types.ts b/packages/core/src/v3/runEngineWorker/supervisor/types.ts index 4d0479cbcac..aff6ffa7d2d 100644 --- a/packages/core/src/v3/runEngineWorker/supervisor/types.ts +++ b/packages/core/src/v3/runEngineWorker/supervisor/types.ts @@ -1,5 +1,13 @@ import type { MachineResources } from "../../schemas/runEngine.js"; +export type SupervisorHttpRequestMetric = { + name: string; + method: string; + status: string; + outcome: "ok" | "http_error" | "invalid_response" | "network_error"; + durationMs: number; +}; + export type SupervisorClientCommonOptions = { apiUrl: string; workerToken: string; @@ -7,6 +15,7 @@ export type SupervisorClientCommonOptions = { deploymentId?: string; managedWorkerSecret?: string; sendRunDebugLogs?: boolean; + onHttpRequestComplete?: (metric: SupervisorHttpRequestMetric) => void; }; export type PreDequeueFn = () => Promise<{ From 9c85e0ecdc22cdf7599b15df4917c0269e4dc321 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 24 Jul 2026 11:52:34 +0100 Subject: [PATCH 04/67] perf(database): index BatchTaskRun on (runtimeEnvironmentId, createdAt, id) for the batches list (#4361) ## Summary The batches list page orders by `createdAt DESC, id DESC` filtered by environment and a created-at window, but the only supporting index on `BatchTaskRun` was `(runtimeEnvironmentId, id)`. That index can't satisfy the `createdAt` ordering, so on environments with a large number of batches the query fell back to a full table scan and in-memory sort, which could run long enough to hit the statement timeout. ## Fix Adds `(runtimeEnvironmentId, createdAt DESC, id DESC)` on `BatchTaskRun`. The query now reads straight from the index in order with no sort step, returning a page with only a handful of heap fetches instead of scanning the whole environment slice. The migration uses `CREATE INDEX CONCURRENTLY IF NOT EXISTS`, so it takes no table lock and is a no-op if the index already exists. --- .server-changes/batch-list-created-at-index.md | 6 ++++++ .../migration.sql | 2 ++ internal-packages/database/prisma/schema.prisma | 1 + 3 files changed, 9 insertions(+) create mode 100644 .server-changes/batch-list-created-at-index.md create mode 100644 internal-packages/database/prisma/migrations/20260724120000_add_batch_task_run_created_at_dashboard_index/migration.sql diff --git a/.server-changes/batch-list-created-at-index.md b/.server-changes/batch-list-created-at-index.md new file mode 100644 index 00000000000..b5a24386614 --- /dev/null +++ b/.server-changes/batch-list-created-at-index.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Speed up the Batches list page for environments with a large number of batches, which could previously time out while loading. diff --git a/internal-packages/database/prisma/migrations/20260724120000_add_batch_task_run_created_at_dashboard_index/migration.sql b/internal-packages/database/prisma/migrations/20260724120000_add_batch_task_run_created_at_dashboard_index/migration.sql new file mode 100644 index 00000000000..4ff6ed34bca --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260724120000_add_batch_task_run_created_at_dashboard_index/migration.sql @@ -0,0 +1,2 @@ +-- CreateIndex +CREATE INDEX CONCURRENTLY IF NOT EXISTS "BatchTaskRun_runtimeEnvironmentId_createdAt_id_idx" ON "public"."BatchTaskRun"("runtimeEnvironmentId", "createdAt" DESC, "id" DESC); diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 2800dd5dd2c..7d6f4ac5493 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -1935,6 +1935,7 @@ model BatchTaskRun { @@index([dependentTaskAttemptId]) // This is for the batch list dashboard page @@index([runtimeEnvironmentId, id(sort: Desc)]) + @@index([runtimeEnvironmentId, createdAt(sort: Desc), id(sort: Desc)]) } enum BatchTaskRunStatus { From 7188eecd83ceef77094e28b6b25804f4631550fe Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 24 Jul 2026 11:56:58 +0100 Subject: [PATCH 05/67] perf(webapp): clamp list-endpoint page size to 100 (#4360) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Several list endpoints accepted an unbounded page size (`perPage` / `per_page` / `pageSize`). An unbounded page lets one request pull an arbitrarily large result set and do a proportional amount of work, which is a poor default for a shared API. This clamps the page size to 100 on every list endpoint that was uncapped, matching the existing cap on `api.v1.runs` and `api.v1.sessions`. Clamping rather than rejecting keeps existing clients working: a request for a larger page returns up to 100 items and offset pagination continues from there. ## Endpoints capped - `api.v1.schedules` (`perPage`) - `api.v1.queues` (`perPage`) - `resources.…versions` (`per_page`) - `resources.…queues` (`per_page`) - `admin.api.v1.…engine.report` (`per_page`) - `admin.api.v1.llm-models` (`pageSize`) Already capped, left as-is: `api.v1.runs`, `api.v1.sessions`, `api.v1.deployments`. --- .server-changes/cap-list-endpoint-page-size.md | 6 ++++++ ...min.api.v1.environments.$environmentId.engine.report.ts | 7 ++++++- apps/webapp/app/routes/admin.api.v1.llm-models.ts | 5 ++++- apps/webapp/app/routes/api.v1.queues.ts | 7 ++++++- apps/webapp/app/routes/api.v1.schedules.ts | 7 ++++++- ...tionSlug.projects.$projectParam.env.$envParam.queues.ts | 6 +++++- ...onSlug.projects.$projectParam.env.$envParam.versions.ts | 6 +++++- 7 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 .server-changes/cap-list-endpoint-page-size.md diff --git a/.server-changes/cap-list-endpoint-page-size.md b/.server-changes/cap-list-endpoint-page-size.md new file mode 100644 index 00000000000..2649f305787 --- /dev/null +++ b/.server-changes/cap-list-endpoint-page-size.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +List API endpoints now clamp the page size to a maximum of 100. Requests asking for a larger page size return up to 100 items and keep paginating, rather than pulling an unbounded page. diff --git a/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.engine.report.ts b/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.engine.report.ts index 7179680d39a..4fdca9475fa 100644 --- a/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.engine.report.ts +++ b/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.engine.report.ts @@ -13,7 +13,12 @@ const ParamsSchema = z.object({ const SearchParamsSchema = z.object({ verbose: z.string().default("0"), page: z.coerce.number().optional(), - per_page: z.coerce.number().optional(), + per_page: z.coerce + .number() + .int() + .positive() + .transform((n) => Math.min(n, 100)) + .optional(), }); export async function loader({ request, params }: LoaderFunctionArgs) { diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.ts index 1d8136ca37e..c35a1d65669 100644 --- a/apps/webapp/app/routes/admin.api.v1.llm-models.ts +++ b/apps/webapp/app/routes/admin.api.v1.llm-models.ts @@ -9,7 +9,10 @@ export async function loader({ request }: LoaderFunctionArgs) { const url = new URL(request.url); const page = parseInt(url.searchParams.get("page") ?? "1"); - const pageSize = parseInt(url.searchParams.get("pageSize") ?? "50"); + const pageSize = Math.max( + 1, + Math.min(parseInt(url.searchParams.get("pageSize") ?? "50") || 50, 100) + ); const [models, total] = await Promise.all([ prisma.llmModel.findMany({ diff --git a/apps/webapp/app/routes/api.v1.queues.ts b/apps/webapp/app/routes/api.v1.queues.ts index 0e976ec8134..b257a248a0b 100644 --- a/apps/webapp/app/routes/api.v1.queues.ts +++ b/apps/webapp/app/routes/api.v1.queues.ts @@ -12,7 +12,12 @@ import { ServiceValidationError } from "~/v3/services/baseService.server"; const SearchParamsSchema = z.object({ page: z.coerce.number().int().positive().optional(), - perPage: z.coerce.number().int().positive().optional(), + perPage: z.coerce + .number() + .int() + .positive() + .transform((n) => Math.min(n, 100)) + .optional(), }); export const loader = createLoaderApiRoute( diff --git a/apps/webapp/app/routes/api.v1.schedules.ts b/apps/webapp/app/routes/api.v1.schedules.ts index 510cf45eacf..b5fd2fd968c 100644 --- a/apps/webapp/app/routes/api.v1.schedules.ts +++ b/apps/webapp/app/routes/api.v1.schedules.ts @@ -12,7 +12,12 @@ import { UpsertTaskScheduleService } from "~/v3/services/upsertTaskSchedule.serv const SearchParamsSchema = z.object({ page: z.coerce.number().int().positive().optional(), - perPage: z.coerce.number().int().positive().optional(), + perPage: z.coerce + .number() + .int() + .positive() + .transform((n) => Math.min(n, 100)) + .optional(), }); export async function action({ request }: ActionFunctionArgs) { diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.ts index 98e31daa4d3..b5c1755eae4 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.ts @@ -9,7 +9,11 @@ import { EnvironmentParamSchema } from "~/utils/pathBuilder"; const SearchParamsSchema = z.object({ query: z.string().optional(), page: z.coerce.number().min(1).default(1), - per_page: z.coerce.number().min(1).default(20), + per_page: z.coerce + .number() + .min(1) + .transform((n) => Math.min(n, 100)) + .default(20), type: z.enum(["task", "custom"]).optional(), }); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.versions.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.versions.ts index f17f2a95c8a..2275ab73bac 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.versions.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.versions.ts @@ -8,7 +8,11 @@ import { EnvironmentParamSchema } from "~/utils/pathBuilder"; const SearchParamsSchema = z.object({ query: z.string().optional(), - per_page: z.coerce.number().min(1).default(25), + per_page: z.coerce + .number() + .min(1) + .transform((n) => Math.min(n, 100)) + .default(25), }); export async function loader({ request, params }: LoaderFunctionArgs) { From bf41c5d5fc5e5f99e1badd916282419396a85a59 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Fri, 24 Jul 2026 12:11:55 +0100 Subject: [PATCH 06/67] feat(supervisor): configurable warm-start dispatch url (#4362) Adds an optional `TRIGGER_WARM_START_DISPATCH_URL`. The warm-start dispatch request uses it when set, otherwise falls back to `TRIGGER_WARM_START_URL`, so the dispatch target can differ from the default warm-start URL. No behavior change when unset. --- apps/supervisor/src/env.ts | 1 + apps/supervisor/src/index.ts | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 6c954b4e76a..2d96cb1e407 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -112,6 +112,7 @@ export const Env = z // Optional services TRIGGER_WARM_START_URL: z.string().optional(), + TRIGGER_WARM_START_DISPATCH_URL: z.string().optional(), TRIGGER_CHECKPOINT_URL: z.string().optional(), TRIGGER_METADATA_URL: z.string().optional(), diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index f30203df547..39b9aa47e0a 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -95,6 +95,8 @@ class ManagedSupervisor { private readonly isKubernetes = isKubernetesEnvironment(env.KUBERNETES_FORCE_ENABLED); private readonly warmStartUrl = env.TRIGGER_WARM_START_URL; + private readonly warmStartDispatchUrl = + env.TRIGGER_WARM_START_DISPATCH_URL ?? env.TRIGGER_WARM_START_URL; private readonly wideEventOpts: WideEventOptions = { service: "supervisor", @@ -698,7 +700,7 @@ class ManagedSupervisor { return false; } - const warmStartUrlWithPath = new URL("/warm-start", this.warmStartUrl); + const warmStartUrlWithPath = new URL("/warm-start", this.warmStartDispatchUrl); const headers: Record = { "Content-Type": "application/json", From 109e245d56e7f001c6ebd8a3ea54a8c31d03bc51 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <209825114+claude[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2026 12:55:31 +0100 Subject: [PATCH 07/67] feat(webapp): show the first and last characters of a new PAT (#4363) --- .../pat-token-reveal-last-chars.md | 6 +++ .../components/primitives/ClipboardField.tsx | 40 ++++++++++++++++++- .../app/routes/account.tokens/route.tsx | 3 ++ 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 .server-changes/pat-token-reveal-last-chars.md diff --git a/.server-changes/pat-token-reveal-last-chars.md b/.server-changes/pat-token-reveal-last-chars.md new file mode 100644 index 00000000000..29797f73d34 --- /dev/null +++ b/.server-changes/pat-token-reveal-last-chars.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +When you create a Personal Access Token, the generated token now shows its first and last few characters instead of being fully hidden, so you can confirm you copied the right value. diff --git a/apps/webapp/app/components/primitives/ClipboardField.tsx b/apps/webapp/app/components/primitives/ClipboardField.tsx index 0304e5bca6b..7774f301218 100644 --- a/apps/webapp/app/components/primitives/ClipboardField.tsx +++ b/apps/webapp/app/components/primitives/ClipboardField.tsx @@ -60,9 +60,43 @@ const variants = { }, }; +const SECURE_MASK = "••••••••••••••••"; + +/** + * Builds the masked display string, optionally revealing the first/last few + * characters in cleartext so users can confirm a copied value. A custom mask + * string (when `secure` is a string) is always shown as-is. + */ +function maskValue( + value: string, + secure: boolean | string, + revealStart: number, + revealEnd: number +) { + if (typeof secure === "string") { + return secure; + } + + const start = Math.max(0, revealStart); + const end = Math.max(0, revealEnd); + + // Nothing to reveal, or revealing would leak the whole value: fully mask. + if ((start === 0 && end === 0) || start + end >= value.length) { + return SECURE_MASK; + } + + const revealedStart = start > 0 ? value.slice(0, start) : ""; + const revealedEnd = end > 0 ? value.slice(-end) : ""; + return `${revealedStart}${SECURE_MASK}${revealedEnd}`; +} + type ClipboardFieldProps = { value: string; secure?: boolean | string; + /** When masked, reveal this many of the first characters in cleartext. */ + secureRevealStart?: number; + /** When masked, reveal this many of the last characters in cleartext. */ + secureRevealEnd?: number; variant: keyof typeof variants; className?: string; icon?: React.ReactNode; @@ -73,6 +107,8 @@ type ClipboardFieldProps = { export function ClipboardField({ value, secure = false, + secureRevealStart = 0, + secureRevealEnd = 0, variant, className, icon, @@ -87,6 +123,8 @@ export function ClipboardField({ setIsSecure(secure !== undefined && secure); }, [secure]); + const maskedValue = maskValue(value, secure, secureRevealStart, secureRevealEnd); + return ( {icon && ( @@ -100,7 +138,7 @@ export function ClipboardField({ } From be45cf9e61d53d7455eaa9a549fe77deb8dd19d2 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 24 Jul 2026 14:20:20 +0100 Subject: [PATCH 08/67] fix(sdk): preserve partial assistant message on chat stream failure (#4348) ## Summary When a `chat.agent` (or `chat.createSession`) turn's model stream fails mid-response (e.g. a transport timeout like `UND_ERR_BODY_TIMEOUT`), the assistant output that already streamed was dropped: `onTurnComplete` fired with `responseMessage: undefined`, and the manual loop's `turn.complete()` rethrew without keeping the partial. Apps that register `hydrateMessages` are hit hardest, since boot-time tail-replay recovery is off by design. This preserves the streamed-so-far assistant output while still reporting the turn as errored, so persistence and recovery keep it. ## Scope of behavior change Only the **error path** changes. Successful turns are unaffected: the same chunks stream to the client in the same order, and backpressure/cancel behave as before. Everything here is a correctness improvement on a turn that hit a source-stream failure. ## What it does Follow-up to #4304 (`chat.pipeAndCapture`), extending the same partial-recovery to the two loops that lacked it: - **`chat.agent`**: taps the response stream (via a `TransformStream`, so pass-through backpressure and cancel are preserved) to buffer chunks, and on a source-stream failure reconstructs the partial (preferring the `onFinish` message). It's surfaced on the error-path `onTurnComplete` (`responseMessage`, `rawResponseMessage`, `uiMessages`, `newUIMessages`, `newMessages`) and committed to the accumulator so the next turn and the reboot snapshot keep it. - **`chat.createSession` / `turn.complete()`**: the reconstructed partial is accumulated (so `turn.uiMessages` reflects it and the caller can persist after catching) before `turn.complete()` rethrows. `onBeforeTurnComplete` stays skipped on the error path (it hands out a writer for a stream that has already broken). ## Correctness properties (each covered by a regression test) Each test below was confirmed to fail without its fix: - The recovered partial reaches `onTurnComplete` and the next turn's accumulated messages. - An already-committed (possibly enriched) response is not overwritten if a post-response hook then throws. - Incomplete tool parts are cleaned from the recovered partial (text kept), so the UI and model views agree and the next turn isn't poisoned. - A prior turn's model-only compaction survives an errored turn (append only the new tail, don't reconvert the full history). - A reconstructed fragment that reuses an existing message id does not clobber the complete message. - Queued `chat.response` data parts are folded into the recovered partial, matching the success path. - `newMessages` (model delta) stays symmetric with `newUIMessages`. ## Tests New `chat-agent-source-stream-error.test.ts` covers the cases above. The full `@trigger.dev/sdk` unit suite passes and the package build is green across all supported runtimes (Node 20 to 26, Bun, Deno, Cloudflare Workers). --- ...gent-preserve-partial-on-stream-failure.md | 5 + packages/trigger-sdk/src/v3/ai.ts | 137 ++++-- .../chat-agent-source-stream-error.test.ts | 435 ++++++++++++++++++ 3 files changed, 551 insertions(+), 26 deletions(-) create mode 100644 .changeset/chat-agent-preserve-partial-on-stream-failure.md create mode 100644 packages/trigger-sdk/test/chat-agent-source-stream-error.test.ts diff --git a/.changeset/chat-agent-preserve-partial-on-stream-failure.md b/.changeset/chat-agent-preserve-partial-on-stream-failure.md new file mode 100644 index 00000000000..3e7defddecd --- /dev/null +++ b/.changeset/chat-agent-preserve-partial-on-stream-failure.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/sdk": patch +--- + +Preserve the partial assistant message when a chat turn's model stream fails mid-response. `chat.agent` now passes the recovered partial to `onTurnComplete`, and `chat.createSession`'s `turn.complete()` keeps it before rethrowing, instead of dropping the streamed-so-far output. diff --git a/packages/trigger-sdk/src/v3/ai.ts b/packages/trigger-sdk/src/v3/ai.ts index 4612302c4ae..0931ff1d89e 100644 --- a/packages/trigger-sdk/src/v3/ai.ts +++ b/packages/trigger-sdk/src/v3/ai.ts @@ -6389,6 +6389,9 @@ function chatAgent< // Declared here so the finally can detach it — a handler leaked past // its turn duplicates every mid-stream message into the shared buffer. let turnMsgSub: { off: () => void } | undefined; + let capturedPartialResponse: TUIMessage | undefined; + let responseCommitted = false; + const turnBufferedChunks: UIMessageChunk[] = []; try { // Extract turn-level context before entering the span. Slim // wire: at most one delta message per record. `headStartMessages` @@ -7175,11 +7178,12 @@ function chatAgent< finishReason?: FinishReason; }) => { capturedResponseMessage = responseMessage as TUIMessage; + capturedPartialResponse = responseMessage as TUIMessage; capturedFinishReason = finishReason; resolveOnFinish!(); }, }); - await pipeChat(uiStream, { + await pipeChat(tapUIMessageChunks(uiStream, turnBufferedChunks), { signal: combinedSignal, spanName: "stream response", }); @@ -7390,6 +7394,12 @@ function chatAgent< } } + if (capturedResponseMessage) { + responseCommitted = true; + capturedPartialResponse = capturedResponseMessage; + turnBufferedChunks.length = 0; + } + if (runSignal.aborted) return "exit"; // Await deferred background work (e.g. DB writes from onTurnStart) @@ -7611,6 +7621,7 @@ function chatAgent< parts: [...(msg.parts ?? []), ...lateParts], } as TUIMessage; capturedResponseMessage = accumulatedUIMessages[idx] as TUIMessage; + capturedPartialResponse = capturedResponseMessage; turnCompleteEvent.responseMessage = capturedResponseMessage; turnCompleteEvent.uiMessages = accumulatedUIMessages; } @@ -7904,10 +7915,77 @@ function chatAgent< ? [...accumulatedUIMessages, erroredWireMessage] : accumulatedUIMessages; - // Fire onTurnComplete on the error path too — the docs promise it - // runs "after every turn, successful or errored" so customers can - // mark the turn failed. `responseMessage` is undefined/partial and - // `error` carries the thrown value. + let partialResponse: TUIMessage | undefined = + capturedPartialResponse ?? + ((await assemblePartialFromChunks(turnBufferedChunks)) as TUIMessage | undefined); + if (partialResponse) { + partialResponse = cleanupAbortedParts(partialResponse); + } + + let partialIdx = partialResponse?.id + ? erroredUIMessages.findIndex((m) => m.id === partialResponse!.id) + : -1; + if (partialResponse && capturedPartialResponse === undefined && partialIdx !== -1) { + partialResponse = undefined; + partialIdx = -1; + } + if (partialResponse && !partialResponse.id) { + partialResponse = { ...partialResponse, id: generateMessageId() } as TUIMessage; + } + if (partialResponse && !responseCommitted) { + const queuedParts = locals.get(chatResponsePartsKey); + if (queuedParts && queuedParts.length > 0) { + partialResponse = { + ...partialResponse, + parts: [...partialResponse.parts, ...(queuedParts as UIMessage["parts"])], + } as TUIMessage; + locals.set(chatResponsePartsKey, []); + } + } + const includePartial = partialResponse != null && !responseCommitted; + let erroredUIMessagesWithPartial: TUIMessage[] = !includePartial + ? erroredUIMessages + : partialIdx === -1 + ? [...erroredUIMessages, partialResponse!] + : (erroredUIMessages.map((m, i) => + i === partialIdx ? partialResponse! : m + ) as TUIMessage[]); + + let erroredNewUIMessages: TUIMessage[] = erroredWireMessage ? [erroredWireMessage] : []; + if (includePartial) { + erroredNewUIMessages.push(partialResponse!); + } + + let erroredNewModelMessages: ModelMessage[] = []; + + if (!responseCommitted) { + try { + if (erroredNewUIMessages.length > 0) { + erroredNewModelMessages = await toModelMessages( + erroredNewUIMessages.map((m) => stripProviderMetadata(m)) + ); + } + if (erroredUIMessagesWithPartial !== accumulatedUIMessages) { + if (partialIdx === -1) { + const appended = erroredUIMessagesWithPartial.slice( + accumulatedUIMessages.length + ); + accumulatedMessages.push( + ...(await toModelMessages(appended.map((m) => stripProviderMetadata(m)))) + ); + } else { + accumulatedMessages = await toModelMessages(erroredUIMessagesWithPartial); + } + accumulatedUIMessages = erroredUIMessagesWithPartial; + locals.set(chatCurrentUIMessagesKey, accumulatedUIMessages); + } + } catch { + erroredNewModelMessages = []; + erroredUIMessagesWithPartial = erroredUIMessages; + erroredNewUIMessages = erroredWireMessage ? [erroredWireMessage] : []; + } + } + if (onTurnComplete) { try { await tracer.startActiveSpan( @@ -7917,11 +7995,11 @@ function chatAgent< ctx, chatId: currentWirePayload.chatId, messages: accumulatedMessages, - uiMessages: erroredUIMessages, - newMessages: [], - newUIMessages: erroredWireMessage ? [erroredWireMessage] : [], - responseMessage: undefined, - rawResponseMessage: undefined, + uiMessages: erroredUIMessagesWithPartial, + newMessages: erroredNewModelMessages, + newUIMessages: erroredNewUIMessages, + responseMessage: partialResponse, + rawResponseMessage: partialResponse, turn, runId: ctx.run.id, chatAccessToken: "", @@ -7967,7 +8045,7 @@ function chatAgent< await writeChatSnapshot(sessionIdForSnapshot, { version: 1, savedAt: Date.now(), - messages: erroredUIMessages, + messages: erroredUIMessagesWithPartial, lastOutEventId: errorTurnCompleteResult?.lastEventId, lastInEventId: errorSnapshotInCursor !== undefined ? String(errorSnapshotInCursor) : undefined, @@ -8887,28 +8965,26 @@ export type PipeAndCaptureResult = { * can return, and propagates a source error to the consumer after buffering * whatever streamed first. See {@link pipeChatAndCapture} for why. */ -async function* tapUIMessageChunks( +function tapUIMessageChunks( source: AsyncIterable | ReadableStream, buffer: UIMessageChunk[] -): AsyncGenerator { +): ReadableStream | AsyncGenerator { if (isReadableStream(source)) { - const reader = source.getReader(); - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - buffer.push(value as UIMessageChunk); - yield value; - } - } finally { - reader.releaseLock(); - } - } else { + return source.pipeThrough( + new TransformStream({ + transform(chunk, controller) { + buffer.push(chunk as UIMessageChunk); + controller.enqueue(chunk); + }, + }) + ); + } + return (async function* () { for await (const chunk of source) { buffer.push(chunk as UIMessageChunk); yield chunk; } - } + })(); } /** @@ -9744,6 +9820,15 @@ function createChatSession( // Surface a genuine stream failure to the caller. A user stop // (status "aborted") falls through so the partial is accumulated. if (captured.status === "error") { + if (captured.message) { + const partial = cleanupAbortedParts(captured.message); + const queuedParts = locals.get(chatResponsePartsKey); + if (queuedParts && queuedParts.length > 0) { + (partial as any).parts = [...(partial.parts ?? []), ...queuedParts]; + locals.set(chatResponsePartsKey, []); + } + await accumulator.addResponse(partial); + } throw captured.error; } response = captured.message; diff --git a/packages/trigger-sdk/test/chat-agent-source-stream-error.test.ts b/packages/trigger-sdk/test/chat-agent-source-stream-error.test.ts new file mode 100644 index 00000000000..0f79b445f71 --- /dev/null +++ b/packages/trigger-sdk/test/chat-agent-source-stream-error.test.ts @@ -0,0 +1,435 @@ +// Import the test harness FIRST — this installs the resource catalog so +// `chat.agent()` calls below register their task functions correctly. +import { mockChatAgent } from "../src/v3/test/index.js"; + +import { describe, expect, it } from "vitest"; +import type { ModelMessage, UIMessage } from "ai"; +import { simulateReadableStream, streamText } from "ai"; +import { MockLanguageModelV3 } from "ai/test"; +import type { LanguageModelV3StreamPart } from "@ai-sdk/provider"; +import { chat } from "../src/v3/ai.js"; +import type { TurnCompleteEvent } from "../src/v3/ai.js"; + +// ── Helpers ──────────────────────────────────────────────────────────── + +function userMessage(text: string, id: string): UIMessage { + return { id, role: "user", parts: [{ type: "text", text }] }; +} + +function extractText(message: UIMessage | undefined): string { + if (!message) return ""; + return (message.parts as Array<{ type: string; text?: string }>) + .filter((p) => p.type === "text") + .map((p) => p.text ?? "") + .join(""); +} + +async function waitFor(check: () => boolean, timeoutMs = 5_000) { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + if (check()) return; + await new Promise((r) => setTimeout(r, 20)); + } + throw new Error("waitFor timed out"); +} + +/** + * A `run()` return value that looks like a `StreamTextResult` (has + * `toUIMessageStream()`) but whose UI stream emits a partial assistant + * message and then errors — reproducing a source-stream transport failure + * (e.g. `UND_ERR_BODY_TIMEOUT`) mid-turn. `onFinish` is never invoked, which + * is exactly what happens on a hard transport error. Chunks are delivered + * one-per-pull before the error so they aren't discarded (calling + * `controller.error()` in the same tick as `enqueue()` resets the queue). + */ +function erroringSource(errorMessage: string) { + const partialChunks = [ + { type: "start", messageId: "a-err" }, + { type: "text-start", id: "t1" }, + { type: "text-delta", id: "t1", delta: "partial answer" }, + ]; + return sourceFromChunks(partialChunks, errorMessage); +} + +function sourceFromChunks(chunks: unknown[], errorMessage: string) { + return { + toUIMessageStream() { + let i = 0; + return new ReadableStream({ + pull(controller) { + if (i < chunks.length) { + controller.enqueue(chunks[i++]); + } else { + controller.error(new Error(errorMessage)); + } + }, + }); + }, + }; +} + +// ── Tests ────────────────────────────────────────────────────────────── + +describe("chat.agent managed loop — source-stream failure", () => { + it("preserves the partial assistant message on onTurnComplete when the source stream fails", async () => { + const turnCompletes: TurnCompleteEvent[] = []; + + const agent = chat.agent({ + id: "chatAgent.source-stream-error", + run: async () => erroringSource("UND_ERR_BODY_TIMEOUT") as never, + onTurnComplete: async (event) => { + turnCompletes.push(event); + }, + }); + + const harness = mockChatAgent(agent, { chatId: "cae-source-error" }); + try { + await harness.sendMessage(userMessage("hi", "u-1")); + await waitFor(() => turnCompletes.length >= 1); + + const evt = turnCompletes[0]!; + + expect(evt.finishReason).toBe("error"); + expect(evt.error).toBeInstanceOf(Error); + expect((evt.error as Error).message).toBe("UND_ERR_BODY_TIMEOUT"); + + expect(evt.responseMessage).toBeDefined(); + expect(extractText(evt.responseMessage)).toBe("partial answer"); + + const newAssistantText = (evt.newMessages as ModelMessage[]) + .filter((m) => m.role === "assistant") + .map((m) => + typeof m.content === "string" + ? m.content + : (m.content as Array<{ type: string; text?: string }>) + .filter((p) => p.type === "text") + .map((p) => p.text ?? "") + .join("") + ) + .join(""); + expect(newAssistantText).toBe("partial answer"); + + expect((evt.newMessages as ModelMessage[]).some((m) => m.role === "user")).toBe(true); + expect((evt.newUIMessages as UIMessage[]).some((m) => m.role === "user")).toBe(true); + } finally { + await harness.close(); + } + }); + + it("carries the recovered partial into the next turn's accumulated messages", async () => { + let turn = 0; + let turn2Messages: ModelMessage[] | undefined; + + const okStream = () => + simulateReadableStream({ + chunks: [ + { type: "text-start", id: "t2" }, + { type: "text-delta", id: "t2", delta: "second answer" }, + { type: "text-end", id: "t2" }, + { + type: "finish", + finishReason: { unified: "stop", raw: "stop" }, + usage: { + inputTokens: { total: 5, noCache: 5, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: 5, text: 5, reasoning: undefined }, + }, + }, + ] as LanguageModelV3StreamPart[], + }); + + const agent = chat.agent({ + id: "chatAgent.source-stream-error-continuation", + run: async ({ messages }) => { + turn++; + if (turn === 1) { + return erroringSource("UND_ERR_BODY_TIMEOUT") as never; + } + turn2Messages = messages; + return streamText({ + model: new MockLanguageModelV3({ doStream: async () => ({ stream: okStream() }) }), + messages, + }); + }, + }); + + const harness = mockChatAgent(agent, { chatId: "cae-source-error-cont" }); + try { + await harness.sendMessage(userMessage("hi", "u-1")); + await harness.sendMessage(userMessage("still there?", "u-2")); + await waitFor(() => turn2Messages !== undefined); + + const assistantText = turn2Messages! + .filter((m) => m.role === "assistant") + .map((m) => + typeof m.content === "string" + ? m.content + : (m.content as Array<{ type: string; text?: string }>) + .filter((p) => p.type === "text") + .map((p) => p.text ?? "") + .join("") + ) + .join(""); + expect(assistantText).toContain("partial answer"); + } finally { + await harness.close(); + } + }); + + it("does not overwrite an already-committed enriched response when a post-response hook throws", async () => { + const events: TurnCompleteEvent[] = []; + + const okModel = (text: string) => + new MockLanguageModelV3({ + doStream: async () => ({ + stream: simulateReadableStream({ + chunks: [ + { type: "text-start", id: "t1" }, + { type: "text-delta", id: "t1", delta: text }, + { type: "text-end", id: "t1" }, + { + type: "finish", + finishReason: { unified: "stop", raw: "stop" }, + usage: { + inputTokens: { + total: 5, + noCache: 5, + cacheRead: undefined, + cacheWrite: undefined, + }, + outputTokens: { total: 5, text: 5, reasoning: undefined }, + }, + }, + ] as LanguageModelV3StreamPart[], + }), + }), + }); + + const agent = chat.agent({ + id: "chatAgent.post-commit-hook-throw", + run: async ({ messages }) => { + chat.response.write({ type: "data-marker", data: { kept: true } } as never); + return streamText({ model: okModel("full response"), messages }); + }, + onTurnComplete: async (event) => { + events.push(event); + if (event.error == null) { + throw new Error("hook boom after commit"); + } + }, + }); + + const harness = mockChatAgent(agent, { chatId: "cae-post-commit-throw" }); + try { + await harness.sendMessage(userMessage("hi", "u-1")); + await waitFor(() => events.some((e) => e.error != null)); + + const errorEvent = events.find((e) => e.error != null)!; + const assistant = (errorEvent.uiMessages as UIMessage[]).find((m) => m.role === "assistant"); + expect(assistant).toBeDefined(); + expect( + (assistant!.parts as Array<{ type: string }>).some((p) => p.type === "data-marker") + ).toBe(true); + expect(extractText(assistant)).toBe("full response"); + } finally { + await harness.close(); + } + }); + + it("does not clobber an existing message when a reconstructed fragment reuses its id", async () => { + let turn = 0; + let firstAssistantId: string | undefined; + const events: TurnCompleteEvent[] = []; + + const okModel = () => + new MockLanguageModelV3({ + doStream: async () => ({ + stream: simulateReadableStream({ + chunks: [ + { type: "text-start", id: "t1" }, + { type: "text-delta", id: "t1", delta: "first answer" }, + { type: "text-end", id: "t1" }, + { + type: "finish", + finishReason: { unified: "stop", raw: "stop" }, + usage: { + inputTokens: { + total: 5, + noCache: 5, + cacheRead: undefined, + cacheWrite: undefined, + }, + outputTokens: { total: 5, text: 5, reasoning: undefined }, + }, + }, + ] as LanguageModelV3StreamPart[], + }), + }), + }); + + const collidingErroringSource = (id: string) => ({ + toUIMessageStream() { + const chunks = [ + { type: "start", messageId: id }, + { type: "text-start", id: "t2" }, + { type: "text-delta", id: "t2", delta: "clobber" }, + ]; + let i = 0; + return new ReadableStream({ + pull(controller) { + if (i < chunks.length) controller.enqueue(chunks[i++]); + else controller.error(new Error("UND_ERR_BODY_TIMEOUT")); + }, + }); + }, + }); + + const agent = chat.agent({ + id: "chatAgent.fragment-id-collision", + run: async ({ messages }) => { + turn++; + if (turn === 1) { + return streamText({ model: okModel(), messages }); + } + return collidingErroringSource(firstAssistantId!) as never; + }, + onTurnComplete: async (event) => { + events.push(event); + if (event.error == null && event.responseMessage) { + firstAssistantId = event.responseMessage.id; + } + }, + }); + + const harness = mockChatAgent(agent, { chatId: "cae-fragment-collision" }); + try { + await harness.sendMessage(userMessage("hi", "u-1")); + await waitFor(() => firstAssistantId !== undefined); + await harness.sendMessage(userMessage("again", "u-2")); + await waitFor(() => events.some((e) => e.error != null)); + + const errorEvent = events.find((e) => e.error != null)!; + const preserved = (errorEvent.uiMessages as UIMessage[]).find( + (m) => m.id === firstAssistantId + ); + expect(preserved).toBeDefined(); + expect(extractText(preserved)).toBe("first answer"); + expect( + (errorEvent.uiMessages as UIMessage[]).some((m) => extractText(m).includes("clobber")) + ).toBe(false); + } finally { + await harness.close(); + } + }); + + it("cleans dangling tool parts from the recovered partial while keeping its text", async () => { + const turnCompletes: TurnCompleteEvent[] = []; + + const agent = chat.agent({ + id: "chatAgent.error-partial-cleanup", + run: async () => + sourceFromChunks( + [ + { type: "start", messageId: "a-tool" }, + { type: "text-start", id: "t1" }, + { type: "text-delta", id: "t1", delta: "thinking" }, + { type: "text-end", id: "t1" }, + { type: "tool-input-start", toolCallId: "tc1", toolName: "search" }, + { + type: "tool-input-available", + toolCallId: "tc1", + toolName: "search", + input: { q: "x" }, + }, + ], + "UND_ERR_BODY_TIMEOUT" + ) as never, + onTurnComplete: async (event) => { + turnCompletes.push(event); + }, + }); + + const harness = mockChatAgent(agent, { chatId: "cae-partial-cleanup" }); + try { + await harness.sendMessage(userMessage("hi", "u-1")); + await waitFor(() => turnCompletes.length >= 1); + + const evt = turnCompletes[0]!; + expect(evt.responseMessage).toBeDefined(); + const parts = evt.responseMessage!.parts as Array<{ type: string }>; + expect(extractText(evt.responseMessage)).toBe("thinking"); + expect(parts.some((p) => p.type.startsWith("tool-"))).toBe(false); + } finally { + await harness.close(); + } + }); + + it("folds queued response data parts into the recovered partial", async () => { + const turnCompletes: TurnCompleteEvent[] = []; + + const agent = chat.agent({ + id: "chatAgent.error-queued-parts", + run: async () => { + chat.response.write({ type: "data-marker", data: { kept: true } } as never); + return erroringSource("UND_ERR_BODY_TIMEOUT") as never; + }, + onTurnComplete: async (event) => { + turnCompletes.push(event); + }, + }); + + const harness = mockChatAgent(agent, { chatId: "cae-error-queued-parts" }); + try { + await harness.sendMessage(userMessage("hi", "u-1")); + await waitFor(() => turnCompletes.length >= 1); + + const evt = turnCompletes[0]!; + expect(evt.responseMessage).toBeDefined(); + const parts = evt.responseMessage!.parts as Array<{ type: string }>; + expect(extractText(evt.responseMessage)).toBe("partial answer"); + expect(parts.some((p) => p.type === "data-marker")).toBe(true); + } finally { + await harness.close(); + } + }); +}); + +describe("chat.createSession turn.complete() — source-stream failure", () => { + it("accumulates the partial before rethrowing so the caller can persist it", async () => { + let caughtError: unknown; + let uiMessagesAfterError: UIMessage[] = []; + + const agent = chat.customAgent({ + id: "createSession.source-stream-error", + run: async (payload) => { + const session = chat.createSession(payload, { + signal: new AbortController().signal, + idleTimeoutInSeconds: 2, + }); + for await (const turn of session) { + try { + await turn.complete(erroringSource("UND_ERR_BODY_TIMEOUT") as never); + } catch (err) { + caughtError = err; + uiMessagesAfterError = [...turn.uiMessages]; + await turn.done(); + } + } + }, + }); + + const harness = mockChatAgent(agent, { chatId: "cs-source-error" }); + try { + await harness.sendMessage(userMessage("hi", "u-1")); + await waitFor(() => caughtError !== undefined); + + expect(caughtError).toBeInstanceOf(Error); + expect((caughtError as Error).message).toBe("UND_ERR_BODY_TIMEOUT"); + + const partial = uiMessagesAfterError.find((m) => m.role === "assistant"); + expect(partial).toBeDefined(); + expect(extractText(partial)).toBe("partial answer"); + } finally { + await harness.close(); + } + }); +}); From d3906241a53e7590095bc26020e87c75409f2029 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Sun, 26 Jul 2026 19:43:41 +0100 Subject: [PATCH 09/67] feat(webapp): read realtime run rows from the primary, not the replica (#4378) ## Summary The realtime runs feed hydrates run rows from read replicas, which means it needs a replica-lag gate to avoid serving a run's previous state right after a write. Setting `REALTIME_BACKEND_NATIVE_RUN_READS_FROM_PRIMARY=1` reads those rows from each run store's primary instead, so there is no lag to gate against: no probe, no wake delay, no stale-read retries. Off by default, so nothing changes unless you set it. ## Design The run stores already decide replica-vs-primary from the *brand* on the read client they are handed: a branded replica keeps the read on the owning store's replica, an unbranded writer escalates it to that store's own primary. So this is a one-line choice at the hydrator, and it stays correct across topologies. With the run-ops split on, each leg lands on its own writer and the caller's client is never forwarded across databases; with the split off, it is the single database's primary. ```ts const runReader = new RunHydrator({ readClient: runReadsFromPrimary ? prisma : $replica, runStore, }); ``` The same flag skips constructing the lag estimator, since probing a replica the feed no longer reads would be measuring the wrong thing. Independently, `AuroraReplicaLagSource` detected Aurora by letting `aurora_replica_status()` fail, on the assumption that the app-level catch made that free. It isn't: an unresolvable function is a query error the driver reports to the error log on every sample, so a non-Aurora replica produced a continuous stream of error events while the estimator quietly fell through to its next candidate. It now resolves the function with `to_regproc` and memoizes the answer, so the unparseable call never reaches the wire. --- .../realtime-run-reads-from-primary.md | 6 +++ apps/webapp/app/env.server.ts | 1 + .../nativeRealtimeClientInstance.server.ts | 8 ++-- .../realtime/replicaLagEstimator.server.ts | 14 ++++++- .../app/services/realtime/runReader.server.ts | 12 +++--- .../shadowRealtimeClientInstance.server.ts | 2 +- .../test/realtime/replicaLagEstimator.test.ts | 41 +++++++++++++++++++ .../test/realtime/runReaderProjection.test.ts | 5 ++- .../realtime/runReaderReadThrough.test.ts | 18 ++++---- .../test/realtimeServices.replicaLag.test.ts | 4 +- 10 files changed, 89 insertions(+), 22 deletions(-) create mode 100644 .server-changes/realtime-run-reads-from-primary.md diff --git a/.server-changes/realtime-run-reads-from-primary.md b/.server-changes/realtime-run-reads-from-primary.md new file mode 100644 index 00000000000..c52ea44078b --- /dev/null +++ b/.server-changes/realtime-run-reads-from-primary.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Realtime run subscriptions can now be configured to read run data straight from the primary database, so a run's latest state is never served from a lagging replica. Off by default; replica reads are unchanged unless you turn it on. diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 53b5edf0a46..2d28827f7d7 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -462,6 +462,7 @@ const EnvironmentSchema = z // TTL/size of the per-org realtimeBackend flag cache used to pick the serving backend. REALTIME_BACKEND_FLAG_CACHE_TTL_MS: z.coerce.number().int().default(30_000), REALTIME_BACKEND_FLAG_CACHE_MAX_ENTRIES: z.coerce.number().int().default(50_000), + REALTIME_BACKEND_NATIVE_RUN_READS_FROM_PRIMARY: z.string().default("0"), // "1" enables the read-your-writes gate: wake hydrates wait out the measured replica lag // (anchored to the change record's updatedAtMs) and stale reads are retried. REALTIME_BACKEND_NATIVE_REPLICA_LAG_GATE_ENABLED: z.string().default("1"), diff --git a/apps/webapp/app/services/realtime/nativeRealtimeClientInstance.server.ts b/apps/webapp/app/services/realtime/nativeRealtimeClientInstance.server.ts index 77a9a02e5fc..b1552a1ec3a 100644 --- a/apps/webapp/app/services/realtime/nativeRealtimeClientInstance.server.ts +++ b/apps/webapp/app/services/realtime/nativeRealtimeClientInstance.server.ts @@ -1,5 +1,5 @@ import { getMeter } from "@internal/tracing"; -import { $replica } from "~/db.server"; +import { $replica, prisma } from "~/db.server"; import { runStore } from "~/v3/runStore.server"; import { env } from "~/env.server"; import { singleton } from "~/utils/singleton"; @@ -130,9 +130,11 @@ function initializeNativeRealtimeClient(): NativeRealtimeClient { }) : undefined; + const runReadsFromPrimary = env.REALTIME_BACKEND_NATIVE_RUN_READS_FROM_PRIMARY === "1"; + // One RunHydrator shared by the router and the client, so its single-flight + short-TTL cache covers both. const runReader = new RunHydrator({ - replica: $replica, + readClient: runReadsFromPrimary ? prisma : $replica, runStore, cacheTtlMs: env.REALTIME_BACKEND_NATIVE_RUN_CACHE_TTL_MS, maxCacheEntries: env.REALTIME_BACKEND_NATIVE_RUN_CACHE_MAX_ENTRIES, @@ -142,7 +144,7 @@ function initializeNativeRealtimeClient(): NativeRealtimeClient { // when idle) and the router delays wake hydrates by it, anchored to each record's // updatedAtMs — so a publish racing the replica's apply is waited out, not read stale. const lagEstimator = - env.REALTIME_BACKEND_NATIVE_REPLICA_LAG_GATE_ENABLED === "1" + !runReadsFromPrimary && env.REALTIME_BACKEND_NATIVE_REPLICA_LAG_GATE_ENABLED === "1" ? new ReplicaLagEstimator({ source: createPostgresReplicaLagSource($replica), sampleIntervalMs: env.REALTIME_BACKEND_NATIVE_REPLICA_LAG_SAMPLE_INTERVAL_MS, diff --git a/apps/webapp/app/services/realtime/replicaLagEstimator.server.ts b/apps/webapp/app/services/realtime/replicaLagEstimator.server.ts index 17aa7e2b893..d077ea32439 100644 --- a/apps/webapp/app/services/realtime/replicaLagEstimator.server.ts +++ b/apps/webapp/app/services/realtime/replicaLagEstimator.server.ts @@ -25,13 +25,25 @@ export interface ReplicaLagSource { /** Aurora: replicas share the storage layer and reject every standard WAL function; * `aurora_replica_status()` is the only live lag source. Max across readers, since the * `$replica` pool balances over all of them. No reader rows = `$replica` is the writer = - * no lag. Throws on non-Aurora (the function doesn't exist). */ + * no lag. Throws on non-Aurora, but detects that with a `to_regproc` lookup rather than by + * letting the call fail — an unresolvable function is a query error the driver reports to the + * error log (and Sentry) regardless of the app-level catch, once per sample. */ export class AuroraReplicaLagSource implements ReplicaLagSource { readonly name = "aurora"; + #available: boolean | undefined; constructor(private readonly db: RawQueryable) {} async sampleLagMs(): Promise { + if (this.#available === undefined) { + const probe = await this.db.$queryRawUnsafe<{ available: boolean | null }[]>( + `SELECT to_regproc('aurora_replica_status') IS NOT NULL AS available` + ); + this.#available = probe[0]?.available === true; + } + if (!this.#available) { + throw new Error("aurora_replica_status() is not available on this database"); + } const rows = await this.db.$queryRawUnsafe<{ lag: number | null }[]>( `SELECT max(replica_lag_in_msec)::float8 AS lag FROM aurora_replica_status() WHERE session_id <> 'MASTER_SESSION_ID' AND replica_lag_in_msec IS NOT NULL` ); diff --git a/apps/webapp/app/services/realtime/runReader.server.ts b/apps/webapp/app/services/realtime/runReader.server.ts index c6a34d7de7b..c215423b1d4 100644 --- a/apps/webapp/app/services/realtime/runReader.server.ts +++ b/apps/webapp/app/services/realtime/runReader.server.ts @@ -82,9 +82,11 @@ export interface RunListResolver { } export type RunHydratorOptions = { - /** A read-replica Prisma client (`$replica`). Always Postgres. */ - replica: Pick; - /** RunStore the reads are routed through; `replica` is passed as the read client. */ + /** The Prisma client handed to the RunStore as the read client. Always Postgres. A branded + * replica (`$replica`) keeps routed reads on each store's replica; an unbranded writer + * (`prisma`) escalates them to each store's own primary. */ + readClient: Pick; + /** RunStore the reads are routed through. */ runStore: RunStore; /** Read-through cache TTL (ms) collapsing duplicate refetches for the same run. Set 0 to disable. Defaults to 250ms. */ cacheTtlMs?: number; @@ -154,7 +156,7 @@ export class RunHydrator { }, select: buildHydratorSelect(skipColumns), }, - this.options.replica as PrismaClientOrTransaction + this.options.readClient as PrismaClientOrTransaction ); return rows as unknown as RealtimeRunRow[]; } @@ -166,7 +168,7 @@ export class RunHydrator { runtimeEnvironmentId: environmentId, }, { select: RUN_HYDRATOR_SELECT }, - this.options.replica as PrismaClientOrTransaction + this.options.readClient as PrismaClientOrTransaction ); return (run ?? null) as RealtimeRunRow | null; diff --git a/apps/webapp/app/services/realtime/shadowRealtimeClientInstance.server.ts b/apps/webapp/app/services/realtime/shadowRealtimeClientInstance.server.ts index 7453b107837..d4803191659 100644 --- a/apps/webapp/app/services/realtime/shadowRealtimeClientInstance.server.ts +++ b/apps/webapp/app/services/realtime/shadowRealtimeClientInstance.server.ts @@ -21,7 +21,7 @@ function initializeShadowRealtimeClient(): ShadowRealtimeClient { }); const comparator = new RealtimeShadowComparator({ - runReader: new RunHydrator({ replica: $replica, runStore }), + runReader: new RunHydrator({ readClient: $replica, runStore }), runListResolver: new ClickHouseRunListResolver({ getClickhouse: (organizationId) => clickhouseFactory.getClickhouseForOrganization(organizationId, "realtime"), diff --git a/apps/webapp/test/realtime/replicaLagEstimator.test.ts b/apps/webapp/test/realtime/replicaLagEstimator.test.ts index 7666ea28efb..a4a233b613d 100644 --- a/apps/webapp/test/realtime/replicaLagEstimator.test.ts +++ b/apps/webapp/test/realtime/replicaLagEstimator.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it } from "vitest"; import { + AuroraReplicaLagSource, FirstSupportedReplicaLagSource, ReplicaLagEstimator, type ReplicaLagSource, @@ -162,3 +163,43 @@ describe("FirstSupportedReplicaLagSource", () => { expect(composed.name).toBe("flaky"); }); }); + +describe("AuroraReplicaLagSource", () => { + function fakeDb(available: boolean) { + const queries: string[] = []; + return { + queries, + db: { + async $queryRawUnsafe(query: string): Promise { + queries.push(query); + if (query.includes("to_regproc")) { + return [{ available: available ? true : null }] as T; + } + return [{ lag: 12.5 }] as T; + }, + }, + }; + } + + it("never puts aurora_replica_status() on the wire when the function is absent", async () => { + const { db, queries } = fakeDb(false); + const aurora = new AuroraReplicaLagSource(db); + + await expect(aurora.sampleLagMs()).rejects.toThrow(/not available/); + await expect(aurora.sampleLagMs()).rejects.toThrow(/not available/); + + expect(queries.filter((q) => q.includes("FROM aurora_replica_status()"))).toHaveLength(0); + expect(queries.filter((q) => q.includes("to_regproc"))).toHaveLength(1); + }); + + it("samples lag on Aurora and probes for the function only once", async () => { + const { db, queries } = fakeDb(true); + const aurora = new AuroraReplicaLagSource(db); + + expect(await aurora.sampleLagMs()).toBe(12.5); + expect(await aurora.sampleLagMs()).toBe(12.5); + + expect(queries.filter((q) => q.includes("to_regproc"))).toHaveLength(1); + expect(queries.filter((q) => q.includes("FROM aurora_replica_status()"))).toHaveLength(2); + }); +}); diff --git a/apps/webapp/test/realtime/runReaderProjection.test.ts b/apps/webapp/test/realtime/runReaderProjection.test.ts index ad6616f5464..7b92e1f96e2 100644 --- a/apps/webapp/test/realtime/runReaderProjection.test.ts +++ b/apps/webapp/test/realtime/runReaderProjection.test.ts @@ -56,7 +56,10 @@ describe("RunHydrator.hydrateByIds column projection", () => { }, } as any; const runStore = new PostgresRunStore({ prisma: replica, readOnlyPrisma: replica }); - return { hydrator: new RunHydrator({ replica, runStore }), getSelect: () => capturedSelect }; + return { + hydrator: new RunHydrator({ readClient: replica, runStore }), + getSelect: () => capturedSelect, + }; } it("projects the SELECT by skipColumns", async () => { diff --git a/apps/webapp/test/realtime/runReaderReadThrough.test.ts b/apps/webapp/test/realtime/runReaderReadThrough.test.ts index cd23d954fed..8146b527957 100644 --- a/apps/webapp/test/realtime/runReaderReadThrough.test.ts +++ b/apps/webapp/test/realtime/runReaderReadThrough.test.ts @@ -218,7 +218,7 @@ describe("RunHydrator read-route through the runStore seam (legacy + new)", () = }); const runStore = makeRoutingShapedStore({ newStore, legacyStore }); - const hydrator = new RunHydrator({ replica: prisma14, runStore }); + const hydrator = new RunHydrator({ readClient: prisma14, runStore }); const rows = await hydrator.hydrateByIds(envId, [newRunId, legacyRunId]); expect(rows.map((r) => r.id).sort()).toEqual([legacyRunId, newRunId].sort()); @@ -267,7 +267,7 @@ describe("RunHydrator read-route through the runStore seam (legacy + new)", () = }); const runStore = makeRoutingShapedStore({ newStore, legacyStore }); - const hydrator = new RunHydrator({ replica: prisma14, runStore }); + const hydrator = new RunHydrator({ readClient: prisma14, runStore }); const row = await hydrator.getRunById(envId, migratedRunId); expect(row?.id).toBe(migratedRunId); @@ -300,7 +300,7 @@ describe("RunHydrator read-route through the runStore seam (legacy + new)", () = }); const runStore = makeRoutingShapedStore({ newStore, legacyStore }); - const hydrator = new RunHydrator({ replica: prisma14, runStore }); + const hydrator = new RunHydrator({ readClient: prisma14, runStore }); const byId = await hydrator.getRunById(envId, oldRunId); expect(byId?.payload).toBe('{"era":"old"}'); @@ -338,7 +338,7 @@ describe("RunHydrator read-route through the runStore seam (legacy + new)", () = // A generic legacy replica would miss the NEW row entirely — the metadata must come off NEW. const runStore = makeRoutingShapedStore({ newStore, legacyStore }); - const hydrator = new RunHydrator({ replica: prisma14, runStore, cacheTtlMs: 0 }); + const hydrator = new RunHydrator({ readClient: prisma14, runStore, cacheTtlMs: 0 }); const snapshot = await hydrator.getRunById(envId, terminalRunId); expect(snapshot?.id).toBe(terminalRunId); @@ -385,7 +385,7 @@ describe("RunHydrator read-route through the runStore seam (legacy + new)", () = // Use a 0ms TTL so each getRunById re-reads through the seam (no cached stale row across the // crossing). Single-flight/TTL are proven separately below. const runStore = makeRoutingShapedStore({ newStore, legacyStore, classify }); - const hydrator = new RunHydrator({ replica: prisma14, runStore, cacheTtlMs: 0 }); + const hydrator = new RunHydrator({ readClient: prisma14, runStore, cacheTtlMs: 0 }); // Before migration: served from LEGACY. const before = await hydrator.getRunById(envId, runId); @@ -434,7 +434,7 @@ describe("RunHydrator single-flight + TTL cache intact across the seam", () => { }); const runStore = makeRoutingShapedStore({ newStore, legacyStore }); - const hydrator = new RunHydrator({ replica: prisma14, runStore, cacheTtlMs: 60_000 }); + const hydrator = new RunHydrator({ readClient: prisma14, runStore, cacheTtlMs: 60_000 }); // Two concurrent calls -> single-flight collapses to ONE underlying read. const [a, b] = await Promise.all([ @@ -466,7 +466,7 @@ describe("RunHydrator single-flight + TTL cache intact across the seam", () => { const missingRunId = newId("missing_run"); const runStore = makeRoutingShapedStore({ newStore, legacyStore }); - const hydrator = new RunHydrator({ replica: prisma14, runStore, cacheTtlMs: 60_000 }); + const hydrator = new RunHydrator({ readClient: prisma14, runStore, cacheTtlMs: 60_000 }); const first = await hydrator.getRunById(envId, missingRunId); expect(first).toBeNull(); @@ -504,7 +504,7 @@ describe("RunHydrator single-DB passthrough (one PostgresRunStore over one clien }); } - const hydrator = new RunHydrator({ replica: prisma, runStore: store, cacheTtlMs: 60_000 }); + const hydrator = new RunHydrator({ readClient: prisma, runStore: store, cacheTtlMs: 60_000 }); // hydrateByIds returns both rows from the single client. const rows = await hydrator.hydrateByIds(envId, [runIdA, runIdB]); @@ -523,7 +523,7 @@ describe("RunHydrator single-DB passthrough (one PostgresRunStore over one clien postgresTest("empty id-set returns [] without touching the store", async ({ prisma }) => { const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); const findRunsSpy = vi.spyOn(store, "findRuns"); - const hydrator = new RunHydrator({ replica: prisma, runStore: store }); + const hydrator = new RunHydrator({ readClient: prisma, runStore: store }); const rows = await hydrator.hydrateByIds("env_none", []); expect(rows).toEqual([]); diff --git a/apps/webapp/test/realtimeServices.replicaLag.test.ts b/apps/webapp/test/realtimeServices.replicaLag.test.ts index 7df81778db9..02bb605afea 100644 --- a/apps/webapp/test/realtimeServices.replicaLag.test.ts +++ b/apps/webapp/test/realtimeServices.replicaLag.test.ts @@ -216,7 +216,7 @@ describe("realtime-svc — replica-lag guards", () => { const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); // The hydrator holds the real store; hydrateByIds passes `options.replica` as the read client. const hydrator = new RunHydrator({ - replica: replica.client as PrismaClient, + readClient: replica.client as PrismaClient, runStore: writerStore, cacheTtlMs: 0, }); @@ -263,7 +263,7 @@ describe("realtime-svc — replica-lag guards", () => { const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); const hydrator = new RunHydrator({ - replica: replica.client as PrismaClient, + readClient: replica.client as PrismaClient, runStore: writerStore, cacheTtlMs: 0, }); From 4d8b5d6f8709ddaf705de08a1215abaa6fe20651 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <209825114+claude[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:51:05 +0000 Subject: [PATCH 10/67] chore(ci): remove per-repo Dependabot alert workflows (#4384) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## ✅ Checklist - [x] I have followed every step in the [contributing guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md) - [x] The PR title follows the convention. - [ ] I ran and tested the code works — n/a, this PR only deletes two workflow files --- ## Summary **Before:** two scheduled workflows in this repo posted Dependabot digests to Slack — a critical-alert check every morning at 08:00 UTC, and a summary of all open alerts on Mondays at 08:00 UTC. **After:** neither runs. This reporting is handled centrally now, so the two in-repo workflows were duplicating it. **How:** deletes `.github/workflows/dependabot-critical-alerts.yml` and `.github/workflows/dependabot-weekly-summary.yml`. Both were self-contained — inline shell, no shared scripts or composite actions — so nothing else in `.github/` referenced them. Dependabot itself is unchanged: `.github/dependabot.yml`, alerts, and version updates all keep working. This removes only the two Slack notifiers. The `ENABLE_DEPENDABOT_ALERTS` repository variable existed only to switch these two workflows off. Nothing else reads it, so it can be removed from the repository settings if it's set. --- ## Testing No runtime code changes — this PR only removes two scheduled workflow files. Verified that nothing else in the repo references either filename, either workflow name, or the `ENABLE_DEPENDABOT_ALERTS` variable. --- ## Changelog Removed the two in-repo scheduled workflows that posted Dependabot digests to Slack. Co-authored-by: Claude --- .../workflows/dependabot-critical-alerts.yml | 87 -------- .../workflows/dependabot-weekly-summary.yml | 210 ------------------ 2 files changed, 297 deletions(-) delete mode 100644 .github/workflows/dependabot-critical-alerts.yml delete mode 100644 .github/workflows/dependabot-weekly-summary.yml diff --git a/.github/workflows/dependabot-critical-alerts.yml b/.github/workflows/dependabot-critical-alerts.yml deleted file mode 100644 index 62d68c30023..00000000000 --- a/.github/workflows/dependabot-critical-alerts.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: Dependabot Critical Alerts - -on: - schedule: - - cron: "0 8 * * *" # Daily 08:00 UTC - workflow_dispatch: - inputs: - severity: - description: "Severity to alert on" - type: choice - options: - - critical - - high - - medium - - low - default: critical - -concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false - -permissions: - contents: read - -jobs: - alert: - name: Post critical alerts - # Set the ENABLE_DEPENDABOT_ALERTS repository variable to 'false' to turn off - # the Dependabot alert/summary notifiers — e.g. forks/mirrors that lack the - # DEPENDABOT_ALERTS_TOKEN / SLACK_BOT_TOKEN secrets. Defaults to enabled. - if: ${{ vars.ENABLE_DEPENDABOT_ALERTS != 'false' }} - runs-on: warp-ubuntu-latest-x64-2x - environment: dependabot-summary - env: - SEVERITY: ${{ inputs.severity || 'critical' }} - steps: - - name: Fetch alerts - id: alerts - env: - GH_TOKEN: ${{ secrets.DEPENDABOT_ALERTS_TOKEN }} - REPO: ${{ github.repository }} - run: | - set -euo pipefail - gh api -X GET "/repos/$REPO/dependabot/alerts" \ - -F state=open -F severity="$SEVERITY" --paginate > pages.json - jq -s 'add' pages.json > alerts.json - TOTAL=$(jq 'length' alerts.json) - echo "total=$TOTAL" >> "$GITHUB_OUTPUT" - if [ "$TOTAL" = "0" ]; then - exit 0 - fi - LIST=$(jq -r ' - map("• <\(.html_url)|#\(.number)> *\(.dependency.package.name)* - \(.security_advisory.summary)") - | join("\n") - ' alerts.json) - { - echo "list<> "$GITHUB_OUTPUT" - - - name: Build Slack payload - if: steps.alerts.outputs.total != '0' - env: - REPO: ${{ github.repository }} - CHANNEL: ${{ vars.SLACK_CHANNEL_ID }} - TOTAL: ${{ steps.alerts.outputs.total }} - LIST: ${{ steps.alerts.outputs.list }} - run: | - jq -n \ - --arg channel "$CHANNEL" \ - --arg repo "$REPO" \ - --arg total "$TOTAL" \ - --arg list "$LIST" \ - --arg severity "$SEVERITY" \ - '{ - channel: $channel, - text: ":bufo-alarma: `\($repo)` - *\($total) open \($severity) alert(s)*\n\($list)\n\n" - }' > payload.json - - - name: Post Slack alert - if: steps.alerts.outputs.total != '0' - uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3 - with: - method: chat.postMessage - token: ${{ secrets.SLACK_BOT_TOKEN }} - payload-file-path: payload.json diff --git a/.github/workflows/dependabot-weekly-summary.yml b/.github/workflows/dependabot-weekly-summary.yml deleted file mode 100644 index fc9eaebcb94..00000000000 --- a/.github/workflows/dependabot-weekly-summary.yml +++ /dev/null @@ -1,210 +0,0 @@ -name: Dependabot Weekly Summary - -on: - schedule: - - cron: "0 8 * * 1" # Mon 08:00 UTC - workflow_dispatch: - -# Single-purpose monitoring workflow; serialise on workflow name only - we never -# want two concurrent summary runs racing to post the same digest. -concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false - -permissions: - contents: read # gh CLI baseline - pull-requests: read # gh pr list (open dependabot PRs) - actions: read # gh run list / view (parse latest dependabot run logs) - -jobs: - summary: - name: Post weekly Dependabot summary - # Set the ENABLE_DEPENDABOT_ALERTS repository variable to 'false' to turn off - # the Dependabot alert/summary notifiers — e.g. forks/mirrors that lack the - # DEPENDABOT_ALERTS_TOKEN / SLACK_BOT_TOKEN secrets. Defaults to enabled. - if: ${{ vars.ENABLE_DEPENDABOT_ALERTS != 'false' }} - runs-on: warp-ubuntu-latest-x64-2x - environment: dependabot-summary - env: - # Severities surface in the actions list when their remaining TTR drops - # below this many days. Override via repo/env var ACTION_THRESHOLD_DAYS. - THRESHOLD_DAYS: ${{ vars.ACTION_THRESHOLD_DAYS || '7' }} - steps: - - name: Fetch alerts and compute summaries - id: alerts - env: - GH_TOKEN: ${{ secrets.DEPENDABOT_ALERTS_TOKEN }} - REPO: ${{ github.repository }} - run: | - if ! gh api -X GET "/repos/$REPO/dependabot/alerts" --paginate > pages.json 2> err.txt; then - echo "total=?" >> "$GITHUB_OUTPUT" - ERR=$(head -c 200 err.txt | tr '\n' ' ') - echo "by_severity=:x: _failed to fetch alerts: ${ERR}_" >> "$GITHUB_OUTPUT" - echo "actions=:x: _alerts unavailable_" >> "$GITHUB_OUTPUT" - exit 0 - fi - jq -s '[.[][] | select(.state == "open")]' pages.json > open.json - - TOTAL=$(jq 'length' open.json) - echo "total=$TOTAL" >> "$GITHUB_OUTPUT" - - if [ "$TOTAL" = "0" ]; then - echo "by_severity=:white_check_mark: No open alerts." >> "$GITHUB_OUTPUT" - echo "actions=_None_" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # Severity breakdown - real newlines so jq --arg in the payload - # builder encodes them as proper \n in JSON (Slack renders as breaks). - BY_SEV=$(jq -r ' - group_by(.security_advisory.severity) - | map({sev: .[0].security_advisory.severity, - count: length, - weight: ({"critical":0,"high":1,"medium":2,"low":3}[.[0].security_advisory.severity])}) - | sort_by(.weight) - | map("• *\(.count)* \(.sev)") - | join("\n") - ' open.json) - { - echo "by_severity<> "$GITHUB_OUTPUT" - - # Actions: alerts within THRESHOLD_DAYS of their TTR (P0=7d, P1=30d, P2=90d, P3=no deadline) - # Grouped by (package, severity); shows earliest deadline per group. - ACTIONS=$(jq -r --argjson threshold "$THRESHOLD_DAYS" ' - [.[] - | (.security_advisory.severity) as $sev - | ({"critical":7,"high":30,"medium":90,"low":null}[$sev]) as $ttr - | select($ttr != null) - | ((now - (.created_at | fromdateiso8601)) / 86400 | floor) as $age - | {pkg: .dependency.package.name, sev: $sev, remaining: ($ttr - $age)} - ] - | group_by([.pkg, .sev]) - | map({pkg: .[0].pkg, sev: .[0].sev, count: length, min_remaining: ([.[].remaining] | min)}) - | map(select(.min_remaining < $threshold)) - | sort_by(.min_remaining) - | if length == 0 then "_None_" - else (map( - "• *\(.pkg)* (\(.sev))" + - (if .count > 1 then " ×\(.count)" else "" end) + " - " + - (if .min_remaining < 0 then "*OVERDUE* by \(-.min_remaining)d" - else "\(.min_remaining)d remaining" end) - ) | join("\n")) - end - ' open.json) - { - echo "actions<> "$GITHUB_OUTPUT" - - - name: Fetch open dependabot PRs - id: prs - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - REPO_URL: https://github.com/${{ github.repository }} - run: | - if ! PR_JSON=$(gh pr list --repo "$REPO" --state open --author "app/dependabot" --json number,title 2> err.txt); then - ERR=$(head -c 200 err.txt | tr '\n' ' ') - echo "list=:x: _failed to fetch PRs: ${ERR}_" >> "$GITHUB_OUTPUT" - exit 0 - fi - LIST=$(echo "$PR_JSON" | jq -r --arg url "$REPO_URL" ' - if length == 0 then "_None_" - else (map("• <\($url)/pull/\(.number)|#\(.number)> \(.title)") | join("\n")) - end - ') - { - echo "list<> "$GITHUB_OUTPUT" - - - name: Find latest npm dependabot run - id: latest - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - run: | - # Repos without a dependabot.yml have no "Dependabot Updates" workflow; - # treat the lookup failure as "no recent run found" rather than failing. - if ! RUN_ID=$(gh run list --repo "$REPO" --workflow "Dependabot Updates" --status success --limit 30 --json databaseId,name --jq 'first(.[] | select(.name | startswith("npm_and_yarn")) | .databaseId) // empty' 2>/dev/null); then - RUN_ID="" - fi - echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" - - - name: Extract stuck deps (only if actions pending) - id: stuck - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - RUN_ID: ${{ steps.latest.outputs.run_id }} - ACTIONS: ${{ steps.alerts.outputs.actions }} - run: | - # Skip the stuck section entirely when nothing in the actions list - # - keeps the digest tidy when there's nothing to actually act on. - if [ "$ACTIONS" = "_None_" ]; then - echo "section=" >> "$GITHUB_OUTPUT" - exit 0 - fi - HEADER=$'\n\n*Couldn\'t auto-fix (need manual `pnpm.overrides`):*\n' - if [ -z "$RUN_ID" ]; then - { - echo "section<> "$GITHUB_OUTPUT" - exit 0 - fi - gh run view "$RUN_ID" --repo "$REPO" --log > log.txt 2>&1 || true - STUCK=$(grep -oE "No update possible for [^[:space:]]+ [0-9][^[:space:]]*" log.txt | sed 's/No update possible for //' | sort -u || true) - if [ -z "$STUCK" ]; then - { - echo "section<> "$GITHUB_OUTPUT" - exit 0 - fi - LIST=$(echo "$STUCK" | awk 'NR>1{printf "\n"} {printf "• *%s* %s", $1, $2}') - { - echo "section<> "$GITHUB_OUTPUT" - - - name: Build Slack payload - env: - REPO: ${{ github.repository }} - CHANNEL: ${{ vars.SLACK_CHANNEL_ID }} - TOTAL: ${{ steps.alerts.outputs.total }} - BY_SEVERITY: ${{ steps.alerts.outputs.by_severity }} - PRS_LIST: ${{ steps.prs.outputs.list }} - ACTIONS: ${{ steps.alerts.outputs.actions }} - STUCK: ${{ steps.stuck.outputs.section }} - run: | - # Build payload via jq so PR titles or error strings containing - # quotes/backslashes/newlines can't break the JSON. - jq -n \ - --arg channel "$CHANNEL" \ - --arg repo "$REPO" \ - --arg total "$TOTAL" \ - --arg by_severity "$BY_SEVERITY" \ - --arg prs_list "$PRS_LIST" \ - --arg actions "$ACTIONS" \ - --arg stuck "$STUCK" \ - --arg threshold "$THRESHOLD_DAYS" \ - '{ - channel: $channel, - text: ":calendar: *Weekly Dependabot summary* - `\($repo)`\n\n*Open alerts (\($total)):*\n\($by_severity)\n\n*Open Dependabot PRs:*\n\($prs_list)\n\n*Actions needed (<\($threshold)d remaining):*\n\($actions)\($stuck)\n\n" - }' > payload.json - - - name: Post Slack summary - uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3 - with: - method: chat.postMessage - token: ${{ secrets.SLACK_BOT_TOKEN }} - payload-file-path: payload.json From d91818f198e74e0eb4c5d09022c2a5fa3f38d031 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <209825114+claude[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:39:43 +0100 Subject: [PATCH 11/67] fix(webapp): remove unawaited task list metrics promises (#4380) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _Requested via [Slack thread](https://triggerdotdev.slack.com/archives/C097ZHVKZFA/p1785082528841609)_ `TaskListPresenter` created promises that nothing ever consumed. Two of the three deferred metrics promises it returned had no reader, no `await` and no `.catch()`, so when the query behind one of them failed the rejection had nowhere to go. ## Before / After **Before** - `TaskListPresenter.call()` returned four things: `tasks`, `activity`, `runningStats` and `durations`. Its only caller reads `tasks` and `runningStats`. - Every load of the tasks page therefore fired two ClickHouse queries whose results were thrown away. - If either of those two queries failed, the resulting promise rejection was unhandled — nothing was awaiting it and nothing had attached an error handler, so it surfaced as an unhandled rejection at the process level rather than as an error anyone could attribute to a request. **After** - `TaskListPresenter.call()` returns `tasks` and `runningStats` only. - Two fewer queries run per tasks-page load. - There is no longer an unconsumed promise that can reject without a handler. `runningStats` is awaited by its caller, so its failures continue to be handled the way they always were. Nothing changes on screen: the tasks page renders `hourlyActivity` and `runningStates`, and neither of the removed values fed either of those. ## How The removed values were verified unreferenced before deleting anything: - `TaskListPresenter` has exactly one caller, `UnifiedTaskListPresenter`, which reads `taskResult.tasks` and `taskResult.runningStats` and nothing else. - No file anywhere in the repo — app code, tests, or type re-exports — reads an `activity` or `durations` field off the presenter's result. - `UnifiedTaskListPresenter` builds its own `unifiedTaskListHourlyActivity` query for the 24h chart the page actually renders, which is what made the presenter's separate 7-day daily activity data redundant. - `getDailyTaskActivity` and `getAverageDurations` on `ClickHouseEnvironmentMetricsRepository` had no callers other than the two lines being deleted, so they and their now-orphaned helpers and types were removed too. Changes: - `apps/webapp/app/presenters/v3/TaskListPresenter.server.ts` — drop the `activity` and `durations` fields (both from the main return and from the no-current-worker early return) and the two repository calls behind them. Drop the unreferenced `TaskActivity` type alias. The "don't await this" comment on the remaining `runningStats` promise now spells out that the caller has to consume it. - `apps/webapp/app/services/environmentMetricsRepository.server.ts` — remove `getDailyTaskActivity` and `getAverageDurations` from the `EnvironmentMetricsRepository` interface and its ClickHouse implementation, along with `fillInDailyTaskActivity` and the `DailyTaskActivity` / `AverageDurations` types. `getCurrentRunningStats` is the control that shows the diagnosis is right. It throws on query failure in exactly the same way as the two removed methods — `if (queryError) throw queryError` — but it never produced an unhandled rejection, because `UnifiedTaskListPresenter` passes its promise into a `Promise.all(...).then(...)` chain that the route then awaits. Same failure mode, opposite outcome, and the only difference is whether anything consumes the promise. Follow-ups, not in this PR: - `AgentListPresenter` returns three sparkline promises in the same shape and they look similarly unconsumed. Left alone here to keep this change reviewable. - With these two callers gone, the `getTaskActivity` and `getAverageDurations` query builders in `@internal/clickhouse` have no remaining callers in this repo. Whether to remove them is a separate call for someone who owns that package. ## ✅ Checklist - [x] I have followed every step in the [contributing guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md) - [x] The PR title follows the convention. - [x] I ran and tested the code works --- ## Testing - `pnpm run typecheck --filter webapp` — passes. This is the meaningful check here: it proves nothing still references the removed fields, methods or types. - `pnpm run format` and `pnpm run lint:fix` — clean, no changes produced. - No test file referenced the removed symbols, so no test needed updating. --- ## Changelog Server-only change, so this carries a `.server-changes/` note rather than a changeset: `.server-changes/task-list-remove-unused-metrics-queries.md`. > The tasks page no longer runs two queries whose results were never displayed, cutting wasted work on every page load and removing a source of hidden server errors --- ## Screenshots _No visual change — the removed data was never rendered._ Co-authored-by: Claude --- ...task-list-remove-unused-metrics-queries.md | 6 + .../presenters/v3/TaskListPresenter.server.ts | 30 +--- .../environmentMetricsRepository.server.ts | 133 ------------------ 3 files changed, 11 insertions(+), 158 deletions(-) create mode 100644 .server-changes/task-list-remove-unused-metrics-queries.md diff --git a/.server-changes/task-list-remove-unused-metrics-queries.md b/.server-changes/task-list-remove-unused-metrics-queries.md new file mode 100644 index 00000000000..23164a205ca --- /dev/null +++ b/.server-changes/task-list-remove-unused-metrics-queries.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +The tasks page no longer runs two queries whose results were never displayed, cutting wasted work on every page load and removing a source of hidden server errors diff --git a/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts b/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts index f62cfef5c76..1541329884f 100644 --- a/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts @@ -6,10 +6,8 @@ import { import { $replica } from "~/db.server"; import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { - type AverageDurations, ClickHouseEnvironmentMetricsRepository, type CurrentRunningStats, - type DailyTaskActivity, } from "~/services/environmentMetricsRepository.server"; import { singleton } from "~/utils/singleton"; import { findCurrentWorkerFromEnvironment } from "~/v3/models/workerDeployment.server"; @@ -21,8 +19,6 @@ export type TaskListItem = { triggerSource: TaskTriggerSource; }; -export type TaskActivity = DailyTaskActivity[string]; - export class TaskListPresenter { constructor(private readonly _replica: PrismaClientOrTransaction) {} @@ -55,9 +51,7 @@ export class TaskListPresenter { if (!currentWorker) { return { tasks: [], - activity: Promise.resolve({} as DailyTaskActivity), runningStats: Promise.resolve({} as CurrentRunningStats), - durations: Promise.resolve({} as AverageDurations), }; } @@ -89,16 +83,10 @@ export class TaskListPresenter { clickhouse, }); - // IMPORTANT: Don't await these, we want to return the promises - // so we can defer the loading of the data - const activity = environmentMetricsRepository.getDailyTaskActivity({ - organizationId, - projectId, - environmentId, - days: 6, // This actually means 7 days, because we want to show the current day too - tasks: slugs, - }); - + // IMPORTANT: Don't await this, we want to return the promise + // so we can defer the loading of the data. The caller is responsible for + // consuming it — an unconsumed promise here would become an unhandled + // rejection if the underlying query fails. const runningStats = environmentMetricsRepository.getCurrentRunningStats({ organizationId, projectId, @@ -107,15 +95,7 @@ export class TaskListPresenter { tasks: slugs, }); - const durations = environmentMetricsRepository.getAverageDurations({ - organizationId, - projectId, - environmentId, - days: 6, - tasks: slugs, - }); - - return { tasks, activity, runningStats, durations }; + return { tasks, runningStats }; } } diff --git a/apps/webapp/app/services/environmentMetricsRepository.server.ts b/apps/webapp/app/services/environmentMetricsRepository.server.ts index 61b53a4d6ca..5ecdf13f20f 100644 --- a/apps/webapp/app/services/environmentMetricsRepository.server.ts +++ b/apps/webapp/app/services/environmentMetricsRepository.server.ts @@ -2,19 +2,9 @@ import { type ClickHouse } from "@internal/clickhouse"; import type { TaskRunStatus } from "@trigger.dev/database"; import { QUEUED_STATUSES } from "~/components/runs/v3/TaskRunStatus"; -export type DailyTaskActivity = Record)[]>; export type CurrentRunningStats = Record; -export type AverageDurations = Record; export interface EnvironmentMetricsRepository { - getDailyTaskActivity(options: { - organizationId: string; - projectId: string; - environmentId: string; - days: number; - tasks: string[]; - }): Promise; - getCurrentRunningStats(options: { organizationId: string; projectId: string; @@ -22,14 +12,6 @@ export interface EnvironmentMetricsRepository { days: number; tasks: string[]; }): Promise; - - getAverageDurations(options: { - organizationId: string; - projectId: string; - environmentId: string; - days: number; - tasks: string[]; - }): Promise; } export type ClickHouseEnvironmentMetricsRepositoryOptions = { @@ -39,45 +21,6 @@ export type ClickHouseEnvironmentMetricsRepositoryOptions = { export class ClickHouseEnvironmentMetricsRepository implements EnvironmentMetricsRepository { constructor(private readonly options: ClickHouseEnvironmentMetricsRepositoryOptions) {} - public async getDailyTaskActivity({ - organizationId, - projectId, - environmentId, - days, - tasks, - }: { - organizationId: string; - projectId: string; - environmentId: string; - days: number; - tasks: string[]; - }): Promise { - if (tasks.length === 0) { - return {}; - } - - const [queryError, activity] = await this.options.clickhouse.taskRuns.getTaskActivity({ - organizationId, - projectId, - environmentId, - days, - }); - - if (queryError) { - throw queryError; - } - - return fillInDailyTaskActivity( - activity.map((a) => ({ - taskIdentifier: a.task_identifier, - status: a.status as TaskRunStatus, - day: new Date(a.day), - count: BigInt(a.count), - })), - days - ); - } - public async getCurrentRunningStats({ organizationId, projectId, @@ -115,82 +58,6 @@ export class ClickHouseEnvironmentMetricsRepository implements EnvironmentMetric tasks ); } - - public async getAverageDurations({ - organizationId, - projectId, - environmentId, - days, - tasks, - }: { - organizationId: string; - projectId: string; - environmentId: string; - days: number; - tasks: string[]; - }): Promise { - if (tasks.length === 0) { - return {}; - } - - const [queryError, durations] = await this.options.clickhouse.taskRuns.getAverageDurations({ - organizationId, - projectId, - environmentId, - days, - }); - - if (queryError) { - throw queryError; - } - - return Object.fromEntries(durations.map((d) => [d.task_identifier, Number(d.duration)])); - } -} - -type TaskActivityResults = Array<{ - taskIdentifier: string; - status: TaskRunStatus; - day: Date; - count: bigint; -}>; - -function fillInDailyTaskActivity(activity: TaskActivityResults, days: number): DailyTaskActivity { - //today with no time - const today = new Date(); - today.setUTCHours(0, 0, 0, 0); - - return activity.reduce((acc, a) => { - let existingTask = acc[a.taskIdentifier]; - - if (!existingTask) { - existingTask = []; - //populate the array with the past 7 days - for (let i = days; i >= 0; i--) { - const day = new Date(today); - day.setUTCDate(today.getDate() - i); - day.setUTCHours(0, 0, 0, 0); - - existingTask.push({ - day: day.toISOString(), - ["COMPLETED_SUCCESSFULLY"]: 0, - } as { day: string } & Record); - } - - acc[a.taskIdentifier] = existingTask; - } - - const dayString = a.day.toISOString(); - const day = existingTask.find((d) => d.day === dayString); - - if (!day) { - return acc; - } - - day[a.status] = Number(a.count); - - return acc; - }, {} as DailyTaskActivity); } type CurrentRunningStatsResults = Array<{ From 72c2b2c650db389f21a41602447cfae3a3400961 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <209825114+claude[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:09:49 +0000 Subject: [PATCH 12/67] chore(deps): bump express-rate-limit and ip-address (#4391) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Before:** `ip-address` resolved twice in `pnpm-lock.yaml` — `8.1.0` under `@jsonhero/json-infer-types`, and `10.0.1` under `express-rate-limit`. **After:** a single `ip-address@10.2.0` entry, shared by both chains. **How:** `express-rate-limit@8.2.1` pinned `ip-address` to an exact version, so the parent itself had to move — `8.5.1` onwards declares a range instead, and `@modelcontextprotocol/sdk` already allows `^8.2.1`, so scoping that parent to `^8.6.0` lets `ip-address` resolve on its own. `@jsonhero/json-infer-types` caps `ip-address` at `^8.1.0` and is already at its latest published release, so that chain gets a scoped override instead of a parent bump. `jsbn` and `sprintf-js` drop out of the tree as a side effect. Both overrides are parent-scoped, so the `cli-v3` chain is deliberately untouched: it resolves `@modelcontextprotocol/sdk` 1.25.2, which declares `express-rate-limit ^7.5.0` and pulls in no `ip-address` at all. `pnpm-lock.yaml` regenerated. `package.json` and `pnpm-lock.yaml` are the only two files changed. Nothing in the repo imports `ip-address` or `express-rate-limit` directly. Both chains are transitive under `apps/webapp` — `@jsonhero/schema-infer` (used by `TestTaskPresenter.server.ts`) and `@vercel/sdk` — so no published `@trigger.dev/*` package is affected. --- ## Testing - `pnpm install --lockfile-only` regenerates cleanly, and `pnpm install --frozen-lockfile --lockfile-only` passes, so the lockfile matches the manifests. - Package churn is limited to the intended set: `express-rate-limit` 8.2.1 to 8.6.0, `ip-address` 8.1.0 and 10.0.1 collapsing to 10.2.0, and `jsbn` / `sprintf-js` removed. No other resolution moved. - `@jsonhero/json-infer-types` only calls `new Address4()` / `new Address6()` inside a try/catch to classify strings. Ran that exact logic against both `8.1.0` and `10.2.0` over 27 inputs (v4, v6, zone IDs, CIDR, IPv4-mapped, malformed, empty, non-strings): identical results in all 27. Both are still CJS named exports in `10.2.0`, with the same `engines` floor. - Drove the real `inferSchema()` path from `@jsonhero/schema-infer` with `ip-address` forced to `10.2.0`; it still detects `ipv4` and `ipv6` formats correctly. - `express-rate-limit` 8.6.0 keeps the same `express` peer range (`>= 4.11`) and the same node floor as 8.2.1. Its new `debug` dependency resolves to a version already present in the tree. - `oxfmt --check` passes on the modified `package.json`. - Both bumped versions clear the repo's `minimumReleaseAge` window; the newest `express-rate-limit` (8.6.1) and `ip-address` (10.2.1+) releases do not yet, which is why this lands on 8.6.0 and 10.2.0. - Not run here: a full monorepo install, typecheck and test suite. No TypeScript changed, and neither package leaks types into ours — `ip-address` is not referenced in `json-infer-types`' or `schema-infer`'s declaration files — so CI should be the judge of the wider suite. --- ## Changelog Routine dependency maintenance, no behaviour change. No changeset or `.server-changes/` entry: the diff touches only the root `package.json` and `pnpm-lock.yaml`, not `packages/*`, `integrations/*`, `apps/webapp/` or `apps/supervisor/`. Co-authored-by: Claude Co-authored-by: nicktrn <55853254+nicktrn@users.noreply.github.com> --- package.json | 4 +++- pnpm-lock.yaml | 42 ++++++++++++++---------------------------- 2 files changed, 17 insertions(+), 29 deletions(-) diff --git a/package.json b/package.json index 3b00102ac83..f5987056204 100644 --- a/package.json +++ b/package.json @@ -139,7 +139,9 @@ "tmp@<0.2.7": "0.2.7", "brace-expansion@<1.1.13": "1.1.13", "brace-expansion@>=2 <2.0.3": "2.0.3", - "brace-expansion@>=5 <5.0.6": "5.0.6" + "brace-expansion@>=5 <5.0.6": "5.0.6", + "@jsonhero/json-infer-types>ip-address": "^10.2.0", + "@modelcontextprotocol/sdk@>=1.26.0>express-rate-limit": "^8.6.0" }, "onlyBuiltDependencies": [ "@depot/cli", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b09dc718a6f..e49d5239ae0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -51,6 +51,8 @@ overrides: brace-expansion@<1.1.13: 1.1.13 brace-expansion@>=2 <2.0.3: 2.0.3 brace-expansion@>=5 <5.0.6: 5.0.6 + '@jsonhero/json-infer-types>ip-address': ^10.2.0 + '@modelcontextprotocol/sdk@>=1.26.0>express-rate-limit': ^8.6.0 patchedDependencies: '@changesets/assemble-release-plan@5.2.4': @@ -10300,8 +10302,8 @@ packages: peerDependencies: express: ^4.11 || 5 || ^5.0.0-beta.1 - express-rate-limit@8.2.1: - resolution: {integrity: sha512-PCZEIEIxqwhzw4KF0n7QF4QqruVTcF73O5kFKUnGOyjbCCgizBBiFaYpd/fnBLUMPw/BWw9OsiN7GgrNYr7j6g==} + express-rate-limit@8.6.0: + resolution: {integrity: sha512-XKJXDsASUOo0LLtFwW5hCcQGH0N4WQc/Rn8/Pvoia+TJFOkkFPvrtW9lZOeeNcxQJspvOIERMwiRLsVFlhHEkA==} engines: {node: '>= 16'} peerDependencies: express: '>= 4.11' @@ -11018,12 +11020,8 @@ packages: resolution: {integrity: sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==} engines: {node: '>=12.22.0'} - ip-address@10.0.1: - resolution: {integrity: sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA==} - engines: {node: '>= 12'} - - ip-address@8.1.0: - resolution: {integrity: sha512-Wz91gZKpNKoXtqvY8ScarKYwhXoK4r/b5QuT+uywe/azv0/nUCo7Bh0IRRI7F9DHR06kJNWtzMGLIbXavngbKA==} + ip-address@10.2.0: + resolution: {integrity: sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==} engines: {node: '>= 12'} ipaddr.js@1.9.1: @@ -11345,9 +11343,6 @@ packages: resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} hasBin: true - jsbn@1.1.0: - resolution: {integrity: sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==} - jsep@1.4.0: resolution: {integrity: sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==} engines: {node: '>= 10.16.0'} @@ -14198,9 +14193,6 @@ packages: sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} - sprintf-js@1.1.2: - resolution: {integrity: sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==} - sql-formatter@15.6.12: resolution: {integrity: sha512-mkpF+RG402P66VMsnQkWewTRzDBWfu9iLbOfxaW/nAKOS/2A9MheQmcU5cmX0D0At9azrorZwpvcBRNNBozACQ==} hasBin: true @@ -18717,7 +18709,7 @@ snapshots: '@jsonhero/json-infer-types@1.2.11': dependencies: - ip-address: 8.1.0 + ip-address: 10.2.0 json5: 2.2.3 jwt-decode: 3.1.2 uuid: 8.3.2 @@ -18901,7 +18893,7 @@ snapshots: eventsource: 3.0.5 eventsource-parser: 3.1.0 express: 5.2.1 - express-rate-limit: 8.2.1(express@5.2.1) + express-rate-limit: 8.6.0(express@5.2.1) hono: 4.12.25 jose: 6.1.3 json-schema-typed: 8.0.2 @@ -25374,10 +25366,13 @@ snapshots: dependencies: express: 5.0.1(supports-color@10.0.0) - express-rate-limit@8.2.1(express@5.2.1): + express-rate-limit@8.6.0(express@5.2.1): dependencies: + debug: 4.4.3(supports-color@10.0.0) express: 5.2.1 - ip-address: 10.0.1 + ip-address: 10.2.0 + transitivePeerDependencies: + - supports-color express@4.20.0: dependencies: @@ -26323,12 +26318,7 @@ snapshots: transitivePeerDependencies: - supports-color - ip-address@10.0.1: {} - - ip-address@8.1.0: - dependencies: - jsbn: 1.1.0 - sprintf-js: 1.1.2 + ip-address@10.2.0: {} ipaddr.js@1.9.1: {} @@ -26590,8 +26580,6 @@ snapshots: dependencies: argparse: 2.0.1 - jsbn@1.1.0: {} - jsep@1.4.0: {} jsesc@3.0.2: {} @@ -30111,8 +30099,6 @@ snapshots: sprintf-js@1.0.3: {} - sprintf-js@1.1.2: {} - sql-formatter@15.6.12: dependencies: argparse: 2.0.1 From 269470fd87ffd7ad636a91c0f8933e3738625ba9 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 27 Jul 2026 13:08:16 +0100 Subject: [PATCH 13/67] feat(webapp): gate SSO on an entitlement instead of the Enterprise plan (#4393) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SSO & Directory Sync settings page decided access by comparing the organization's plan code against the literal string `"enterprise"`. The webapp now reads a `hasSso` entitlement from plan limits. ## Changes - **`settings.sso` route** — `planAllowsSso` reads `limits.hasSso` rather than the plan code; the loader and the action gate on a shared `getSsoEntitlement` helper. - **`platform.v3.server`** — new `getSsoEntitlement(orgId)` returning `entitled | not_entitled | unknown`, behind a new SWR cache namespace (60s fresh / 120s stale, memory + Redis). This replaces an uncached billing round-trip that previously ran on every settings load, so the page gets cheaper than it was. - **`directorySyncEffects`** — the entitlement is now checked before applying membership effects, per organization and memoised across a batch. - **`@trigger.dev/platform` 1.2.0 → 1.3.0** — required, see below. ## Behaviour worth reviewing **Revocation now stops SCIM.** Previously the plan check existed only on the settings page, so an org that lost access kept receiving directory-sync pushes indefinitely; only the config UI froze. Provision *and* deprovision are gated, so a revoked entitlement can't remove members either. **An unreadable entitlement throws instead of skipping.** Effects are idempotent and the worker retries, so retrying is lossless where dropping would silently lose a directory change. It's raised at `warn` level so a transient billing blip doesn't page anyone. **The login path is deliberately untouched.** A hard entitlement check there turns a billing outage into a login outage. Consequence: an org that loses the entitlement keeps its existing SSO logins working until the connection is removed. Gating sign-in is a separate decision. **Self-hosted is unaffected.** With no billing service configured the helper returns `entitled`, leaving plugin presence and the kill switch as the only gates — a self-hoster who installed the plugin isn't locked out of it. ## The dependency bump is load-bearing The `Limits` schema is a plain `z.object`, so it *strips* unknown keys. On 1.2.0 the `hasSso` field was silently discarded during parsing and read as `undefined` no matter what billing sent — a structural accessor would not have helped. Verified against both builds: ``` 1.2.0 → parsed: true | hasSso survives: false 1.3.0 → parsed: true | hasSso survives: true ``` This PR therefore cannot merge before 1.3.0 is published, which it now is. ## Testing `apps/webapp/test/directorySyncEffects.server.test.ts` — 7 tests over the gate: applies when entitled, skips provision and deprovision when not, throws a warn-level retryable error when unreadable, resolves once per org across a batch, and gates per org so one unentitled org doesn't block another. `pnpm run typecheck --filter webapp` passes (18/18), oxfmt and oxlint clean. --- .server-changes/sso-entitlement.md | 6 + .../route.tsx | 26 ++-- .../services/directorySyncEffects.server.ts | 30 ++++ .../webapp/app/services/platform.v3.server.ts | 68 ++++++++- apps/webapp/package.json | 2 +- .../test/directorySyncEffects.server.test.ts | 130 ++++++++++++++++++ pnpm-lock.yaml | 10 +- 7 files changed, 245 insertions(+), 27 deletions(-) create mode 100644 .server-changes/sso-entitlement.md create mode 100644 apps/webapp/test/directorySyncEffects.server.test.ts diff --git a/.server-changes/sso-entitlement.md b/.server-changes/sso-entitlement.md new file mode 100644 index 00000000000..4a039b6855e --- /dev/null +++ b/.server-changes/sso-entitlement.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +SSO and Directory Sync are no longer restricted to Enterprise plans — get in touch and we can turn them on for your organization whatever plan you're on. diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.sso/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.sso/route.tsx index e58afdb65df..46abc2d4f2b 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.sso/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.sso/route.tsx @@ -39,7 +39,7 @@ import { prisma } from "~/db.server"; import { useOrganization } from "~/hooks/useOrganizations"; import { rbac } from "~/services/rbac.server"; import { ssoController } from "~/services/sso.server"; -import { getCurrentPlan } from "~/services/platform.v3.server"; +import { getSsoEntitlement } from "~/services/platform.v3.server"; import type { DirectorySyncEffect, DirectorySyncStatus, Role } from "@trigger.dev/plugins"; import { applyDirectorySyncEffects } from "~/services/directorySyncEffects.server"; import { flag } from "~/v3/featureFlags.server"; @@ -47,7 +47,6 @@ import { FEATURE_FLAG } from "~/v3/featureFlags"; import { dashboardAction, dashboardLoader } from "~/services/routeBuilders/dashboardBuilder"; import { cn } from "~/utils/cn"; import { throwPermissionDenied } from "~/utils/permissionDenied"; -import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; export const meta: MetaFunction = () => [{ title: "SSO & Directory Sync | Trigger.dev" }]; @@ -62,17 +61,10 @@ async function resolveOrg(slug: string) { }); } -function planAllowsSso(plan: unknown): boolean { - if (!plan || typeof plan !== "object") return false; - const subscription = (plan as { v3Subscription?: { plan?: { code?: string } } }).v3Subscription; - return subscription?.plan?.code === "enterprise"; -} - // Client-side upsell is cosmetic; gate real IdP mutations server-side. async function requireSsoEntitlement(orgId: string): Promise { - const plan = await getCurrentPlan(orgId); - if (!planAllowsSso(plan)) { - throw new Response("SSO requires an Enterprise plan", { status: 403 }); + if ((await getSsoEntitlement(orgId)) !== "entitled") { + throw new Response("This organization is not entitled to SSO", { status: 403 }); } } @@ -142,16 +134,14 @@ export const loader = dashboardLoader( throw new Response("Not Found", { status: 404 }); } - // Not Enterprise: render the upsell for every role, skip role check + - // queries, return empty data. - const plan = await getCurrentPlan(orgId); - if (!planAllowsSso(plan)) { + if ((await getSsoEntitlement(orgId)) !== "entitled") { return typedjson({ status: EMPTY_SSO_STATUS, orgTitle: context.orgTitle, jitRoles: [] as Role[], directorySync: EMPTY_DIRECTORY_SYNC_STATUS, hasSso: false, + isEntitled: false, }); } @@ -182,6 +172,7 @@ export const loader = dashboardLoader( jitRoles, directorySync, hasSso, + isEntitled: true, }); } ); @@ -374,11 +365,10 @@ function useOverrideDraft(serverValue: T): { } export default function Page() { - const { status, orgTitle, jitRoles, directorySync, hasSso } = useTypedLoaderData(); + const { status, orgTitle, jitRoles, directorySync, hasSso, isEntitled } = + useTypedLoaderData(); const organization = useOrganization(); - const _plan = useCurrentPlan(); - const isEntitled = planAllowsSso(_plan); const activeConnections = status.connections.filter((c) => c.state === "active"); const hasActive = activeConnections.length > 0; diff --git a/apps/webapp/app/services/directorySyncEffects.server.ts b/apps/webapp/app/services/directorySyncEffects.server.ts index 24e1dade5a2..555bb6f98ab 100644 --- a/apps/webapp/app/services/directorySyncEffects.server.ts +++ b/apps/webapp/app/services/directorySyncEffects.server.ts @@ -8,6 +8,7 @@ import { removeOrgMemberForDirectory, } from "~/models/orgMember.server"; import { createPlatformNotification } from "~/services/platformNotifications.server"; +import { getSsoEntitlement, type SsoEntitlement } from "~/services/platform.v3.server"; const LAST_OWNER_NOTIFICATION_TITLE = "Directory Sync: last Owner protected"; @@ -145,8 +146,37 @@ async function applyEffect(effect: DirectorySyncEffect): Promise { } } +/** + * Applies membership effects, skipping any org that isn't entitled to SSO. + * + * An unreadable entitlement throws rather than skipping: effects are + * idempotent and the worker retries, so retrying is lossless where dropping + * would silently lose a directory change. + */ export async function applyDirectorySyncEffects(effects: DirectorySyncEffect[]): Promise { + const entitlements = new Map(); + for (const effect of effects) { + let entitlement = entitlements.get(effect.organizationId); + if (entitlement === undefined) { + entitlement = await getSsoEntitlement(effect.organizationId); + entitlements.set(effect.organizationId, entitlement); + } + + if (entitlement === "unknown") { + throw retryableEffectError( + `directory sync: could not read the SSO entitlement for organization ${effect.organizationId}` + ); + } + + if (entitlement === "not_entitled") { + logger.warn("Directory Sync: skipping effect for org without the SSO entitlement", { + organizationId: effect.organizationId, + kind: effect.kind, + }); + continue; + } + await applyEffect(effect); } } diff --git a/apps/webapp/app/services/platform.v3.server.ts b/apps/webapp/app/services/platform.v3.server.ts index bded0b92065..f8b524e3cdd 100644 --- a/apps/webapp/app/services/platform.v3.server.ts +++ b/apps/webapp/app/services/platform.v3.server.ts @@ -196,6 +196,11 @@ function initializePlatformCache() { fresh: 60_000, stale: 120_000, }), + ssoEntitlement: new Namespace(ctx, { + stores: [memory, redisCacheStore], + fresh: 60_000, + stale: 120_000, + }), }); return cache; @@ -206,12 +211,23 @@ const platformCache = singleton("platformCache", initializePlatformCache); function invalidateBillingLimitCaches(organizationId: string) { platformCache.billingLimit.remove(organizationId).catch(() => {}); platformCache.entitlement.remove(organizationId).catch(() => {}); + platformCache.ssoEntitlement.remove(organizationId).catch(() => {}); } export function bustBillingLimitCaches(organizationId: string) { invalidateBillingLimitCaches(organizationId); } +/** + * Clears the caches whose value is derived from the org's plan. Call after a + * plan change — a downgrade can revoke SSO, and serving the previous decision + * for the stale TTL would keep a surface open that the new plan doesn't allow. + */ +function invalidatePlanDerivedCaches(organizationId: string) { + platformCache.entitlement.remove(organizationId).catch(() => {}); + platformCache.ssoEntitlement.remove(organizationId).catch(() => {}); +} + // Clear the cached promo-credits read so a just-granted code shows on the usage // page immediately rather than after the stale TTL. export function bustPromoCreditsCache(organizationId: string) { @@ -537,7 +553,7 @@ export async function setPlan( case "free_connected": { // Selecting Free provisions the plan directly, so any free result is a success. opts?.invalidateBillingCache?.(organization.id); - platformCache.entitlement.remove(organization.id).catch(() => {}); + invalidatePlanDerivedCaches(organization.id); const response = redirect(newProjectPath(organization, "You're on the Free plan.")); await opts?.onFreePlanProvisioned?.(response); return response; @@ -548,13 +564,13 @@ export async function setPlan( case "updated_subscription": { // Invalidate billing cache since subscription changed opts?.invalidateBillingCache?.(organization.id); - platformCache.entitlement.remove(organization.id).catch(() => {}); + invalidatePlanDerivedCaches(organization.id); return redirectWithSuccessMessage(callerPath, request, "Subscription updated successfully."); } case "canceled_subscription": { // Invalidate billing cache since subscription was canceled opts?.invalidateBillingCache?.(organization.id); - platformCache.entitlement.remove(organization.id).catch(() => {}); + invalidatePlanDerivedCaches(organization.id); return redirectWithSuccessMessage(callerPath, request, "Subscription canceled."); } } @@ -757,6 +773,52 @@ export async function getEntitlement( return result.val; } +export type SsoEntitlement = "entitled" | "not_entitled" | "unknown"; + +/** + * Whether an org may configure and use SSO / Directory Sync. + * + * `unknown` means billing was configured but unreadable — callers decide: + * read paths show the upsell, mutations refuse, and the directory-sync + * worker throws so the effect is retried rather than silently dropped. + * + * Self-hosted deployments have no billing service, so the plugin's presence + * (plus the kill switch) is the only gate and this returns `entitled`. + * + * Loader errors are swallowed inside the loader for the same reason as + * `getEntitlement`: @unkey/cache passes the loader promise to waitUntil() + * with no .catch(), and returning undefined stops a transient billing + * failure from being cached as an access decision. The SWR read is guarded + * too, so a cache-infra failure resolves to `unknown` rather than rejecting + * into the settings loader and the directory-sync worker. + */ +export async function getSsoEntitlement(organizationId: string): Promise { + if (!client) return "entitled"; + + try { + const result = await platformCache.ssoEntitlement.swr(organizationId, async () => { + try { + const response = await client.currentPlan(organizationId); + if (!response.success) { + recordPlatformFailure("getSsoEntitlement", "no_success"); + return undefined; + } + return response.v3Subscription?.plan?.limits?.hasSso === true; + } catch (_e) { + recordPlatformFailure("getSsoEntitlement", "caught"); + return undefined; + } + }); + + if (result.err || result.val === undefined) return "unknown"; + + return result.val ? "entitled" : "not_entitled"; + } catch (_e) { + recordPlatformFailure("getSsoEntitlement", "caught"); + return "unknown"; + } +} + export type PromoCreditsData = { grantedCents: number; remainingCents: number; diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 678eeb76033..0eb149f77ed 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -117,7 +117,7 @@ "@trigger.dev/core": "workspace:*", "@trigger.dev/database": "workspace:*", "@trigger.dev/otlp-importer": "workspace:*", - "@trigger.dev/platform": "1.2.0", + "@trigger.dev/platform": "1.3.0", "@trigger.dev/plugins": "workspace:*", "@trigger.dev/rbac": "workspace:*", "@trigger.dev/redis-worker": "workspace:*", diff --git a/apps/webapp/test/directorySyncEffects.server.test.ts b/apps/webapp/test/directorySyncEffects.server.test.ts new file mode 100644 index 00000000000..e8f9bb933cc --- /dev/null +++ b/apps/webapp/test/directorySyncEffects.server.test.ts @@ -0,0 +1,130 @@ +import type { DirectorySyncEffect } from "@trigger.dev/plugins"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} })); +vi.mock("~/services/platformNotifications.server", () => ({ + createPlatformNotification: vi.fn(), +})); + +const getSsoEntitlement = vi.fn(); +vi.mock("~/services/platform.v3.server", async (importOriginal) => { + const actual = (await importOriginal()) as Record; + return { ...actual, getSsoEntitlement: (orgId: string) => getSsoEntitlement(orgId) }; +}); + +const setUserRole = vi.fn(); +vi.mock("~/services/rbac.server", () => ({ + rbac: { setUserRole: (a: unknown) => setUserRole(a) }, +})); + +const ensureOrgMember = vi.fn(); +const ensureUserForDirectory = vi.fn(); +const removeOrgMemberForDirectory = vi.fn(); +vi.mock("~/models/orgMember.server", () => ({ + ensureOrgMember: (a: unknown) => ensureOrgMember(a), + ensureUserForDirectory: (a: unknown) => ensureUserForDirectory(a), + removeOrgMemberForDirectory: (a: unknown) => removeOrgMemberForDirectory(a), +})); + +import { applyDirectorySyncEffects } from "~/services/directorySyncEffects.server"; + +const ENTITLED_ORG = "org_entitled"; +const UNENTITLED_ORG = "org_unentitled"; + +function provision(organizationId: string, email = "someone@acme.com"): DirectorySyncEffect { + return { + kind: "provision", + userId: "user_1", + email, + firstName: null, + lastName: null, + organizationId, + roleId: null, + }; +} + +function deprovision(organizationId: string): DirectorySyncEffect { + return { kind: "deprovision", userId: "user_1", organizationId }; +} + +describe("applyDirectorySyncEffects — SSO entitlement gate", () => { + beforeEach(() => { + vi.clearAllMocks(); + ensureOrgMember.mockResolvedValue(undefined); + removeOrgMemberForDirectory.mockResolvedValue({ removed: true }); + setUserRole.mockResolvedValue({ ok: true }); + }); + + it("applies effects for an entitled org", async () => { + getSsoEntitlement.mockResolvedValue("entitled"); + + await applyDirectorySyncEffects([provision(ENTITLED_ORG)]); + + expect(ensureOrgMember).toHaveBeenCalledTimes(1); + expect(ensureOrgMember).toHaveBeenCalledWith( + expect.objectContaining({ organizationId: ENTITLED_ORG, source: "directory_sync" }) + ); + }); + + it("skips provisioning for an org without the entitlement", async () => { + getSsoEntitlement.mockResolvedValue("not_entitled"); + + await applyDirectorySyncEffects([provision(UNENTITLED_ORG)]); + + expect(ensureOrgMember).not.toHaveBeenCalled(); + expect(ensureUserForDirectory).not.toHaveBeenCalled(); + }); + + it("skips deprovisioning too, so revocation cannot remove members", async () => { + getSsoEntitlement.mockResolvedValue("not_entitled"); + + await applyDirectorySyncEffects([deprovision(UNENTITLED_ORG)]); + + expect(removeOrgMemberForDirectory).not.toHaveBeenCalled(); + }); + + it("throws on an unreadable entitlement so the worker retries", async () => { + getSsoEntitlement.mockResolvedValue("unknown"); + + await expect(applyDirectorySyncEffects([provision(ENTITLED_ORG)])).rejects.toThrow( + /could not read the SSO entitlement/ + ); + + expect(ensureOrgMember).not.toHaveBeenCalled(); + }); + + it("marks the retry as a warning rather than a pageable error", async () => { + getSsoEntitlement.mockResolvedValue("unknown"); + + await applyDirectorySyncEffects([provision(ENTITLED_ORG)]).then( + () => expect.unreachable("should have thrown"), + (error) => expect(error).toMatchObject({ logLevel: "warn" }) + ); + }); + + it("resolves the entitlement once per org across a batch", async () => { + getSsoEntitlement.mockResolvedValue("entitled"); + + await applyDirectorySyncEffects([ + provision(ENTITLED_ORG, "a@acme.com"), + provision(ENTITLED_ORG, "b@acme.com"), + provision(ENTITLED_ORG, "c@acme.com"), + ]); + + expect(getSsoEntitlement).toHaveBeenCalledTimes(1); + expect(ensureOrgMember).toHaveBeenCalledTimes(3); + }); + + it("gates per org, so one unentitled org does not block another", async () => { + getSsoEntitlement.mockImplementation(async (orgId: string) => + orgId === ENTITLED_ORG ? "entitled" : "not_entitled" + ); + + await applyDirectorySyncEffects([provision(UNENTITLED_ORG), provision(ENTITLED_ORG)]); + + expect(ensureOrgMember).toHaveBeenCalledTimes(1); + expect(ensureOrgMember).toHaveBeenCalledWith( + expect.objectContaining({ organizationId: ENTITLED_ORG }) + ); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e49d5239ae0..204cae651c0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -480,8 +480,8 @@ importers: specifier: workspace:* version: link:../../internal-packages/otlp-importer '@trigger.dev/platform': - specifier: 1.2.0 - version: 1.2.0(zod@3.25.76) + specifier: 1.3.0 + version: 1.3.0(zod@3.25.76) '@trigger.dev/plugins': specifier: workspace:* version: link:../../packages/plugins @@ -7683,8 +7683,8 @@ packages: react: 18.3.1 react-dom: 18.3.1 - '@trigger.dev/platform@1.2.0': - resolution: {integrity: sha512-Wa/XlMlmo1vhol5DEBYW1gMW4wgUUqr/ClDQb4IgwrRdPeLLmTpIVsgMHmYJXkERqR04xVaX7wBibWl4sW+1Hg==} + '@trigger.dev/platform@1.3.0': + resolution: {integrity: sha512-DIJ5Sy4X5XKSHQdYA3GImkb0jdE9I/HTRRRB3c9CFuXjGzhK3rJkEzjIEbaNRjk6yAev67EVTWY8P+D+khorUw==} peerDependencies: zod: ^3.25.0 || ^4.0.0 @@ -22326,7 +22326,7 @@ snapshots: react: 18.3.1 react-dom: 18.3.1(react@18.3.1) - '@trigger.dev/platform@1.2.0(zod@3.25.76)': + '@trigger.dev/platform@1.3.0(zod@3.25.76)': dependencies: zod: 3.25.76 From e8a2dbd6053d195d738d270ac830d81c8e20f458 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:44:14 +0100 Subject: [PATCH 14/67] chore: ignore local docs/superpowers planning docs (#4395) Adds a gitignore rule for `**/docs/superpowers/` so locally-generated planning and design scratch docs under that path aren't committed; preventive only, no-op for existing tree. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 1ee5643d57d..7d9dc169042 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,6 @@ ailogger-output.log # local git worktree checkouts (not source) — keeps oxfmt/oxlint from descending into them .worktrees/ + +# local planning/design docs, not committed +**/docs/superpowers/ From 3e53404f408083a91ba923c9fb831bd965942461 Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Mon, 27 Jul 2026 15:41:47 +0100 Subject: [PATCH 15/67] feat(webapp): live-update the runs list on task pages (#4377) ## Summary The runs list on a task's page now updates live, matching the main Runs page. Run rows update their status, duration, and cost in place as runs progress, and a "N new runs" button appears in the header when newer runs come in so you can pull them into the list without a manual refresh. This applies to both standard and scheduled task pages. ## Design It reuses the Runs page's polling hook. A task page scopes its runs by the task in the URL path rather than a `tasks` query filter, so the hook now takes an optional task slug and scopes new-run detection to it. The "new runs" button sits in the header, outside the deferred runs table, so the count is lifted to the page and the click action is passed through a ref. That keeps the table streaming on first load instead of blocking the header on the runs query. When newer runs come in, a `1 new run` button appears in the task page header, to the left of the time filter. Clicking it pulls the new runs into the list. --- .../task-page-runs-live-updating.md | 6 + .../app/components/runs/v3/TaskRunsList.tsx | 117 ++++++++++++++++++ .../useRunsLiveReload.ts | 20 ++- .../route.tsx | 30 +++-- .../route.tsx | 32 +++-- 5 files changed, 179 insertions(+), 26 deletions(-) create mode 100644 .server-changes/task-page-runs-live-updating.md create mode 100644 apps/webapp/app/components/runs/v3/TaskRunsList.tsx diff --git a/.server-changes/task-page-runs-live-updating.md b/.server-changes/task-page-runs-live-updating.md new file mode 100644 index 00000000000..a082fdfb3d5 --- /dev/null +++ b/.server-changes/task-page-runs-live-updating.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +The runs list on a task's page now updates live — run statuses change and newly triggered runs appear without a manual refresh, matching the main Runs page. diff --git a/apps/webapp/app/components/runs/v3/TaskRunsList.tsx b/apps/webapp/app/components/runs/v3/TaskRunsList.tsx new file mode 100644 index 00000000000..1b9830ec54f --- /dev/null +++ b/apps/webapp/app/components/runs/v3/TaskRunsList.tsx @@ -0,0 +1,117 @@ +import { useLocation, useNavigation, useRevalidator } from "@remix-run/react"; +import { type MutableRefObject, useEffect } from "react"; +import { Button } from "~/components/primitives/Buttons"; +import { PulsingDot } from "~/components/primitives/PulsingDot"; +import { useEnvironment } from "~/hooks/useEnvironment"; +import { useOrganization } from "~/hooks/useOrganizations"; +import { useProject } from "~/hooks/useProject"; +import { useSearchParams } from "~/hooks/useSearchParam"; +import type { NextRunList } from "~/presenters/v3/NextRunListPresenter.server"; +import { useRunsLiveReload } from "~/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/useRunsLiveReload"; +import { TaskRunsTable } from "./TaskRunsTable"; + +/** + * Compact "N new runs" button, shown in a task page's header to the left of the + * time filter when the live-reload hook has detected newer runs. + */ +export function NewRunsButton({ count, onClick }: { count: number; onClick: () => void }) { + return ( + + + + ); +} + +/** + * Runs table with live updating, shared by the standard and scheduled task + * landing pages. Mirrors the Runs list page: active rows are patched in place + * (status/timing/cost). The "N new runs" count is surfaced to the top-bar + * button via `onNewRunsCountChange` (count drives visibility) and + * `showNewRunsRef` (the latest click action), since the button lives outside + * this deferred boundary. The task lives in the route path rather than a + * `tasks` filter, so we pass `taskSlug` to scope new-run detection to this task. + */ +export function TaskRunsList({ + list, + taskSlug, + onNewRunsCountChange, + showNewRunsRef, +}: { + list: NextRunList; + taskSlug: string; + onNewRunsCountChange: (count: number) => void; + showNewRunsRef: MutableRefObject<() => void>; +}) { + const organization = useOrganization(); + const project = useProject(); + const environment = useEnvironment(); + const navigation = useNavigation(); + const location = useLocation(); + const { has, replace } = useSearchParams(); + const revalidator = useRevalidator(); + + // Loading a new version of this same page (time filter / pagination change). + const isLoading = + navigation.state === "loading" && + navigation.location !== undefined && + navigation.location.pathname === location.pathname && + navigation.location.search !== location.search; + + const { visibleRuns, newRunsCount, dismissNewRuns, childrenStatusesBasePath } = useRunsLiveReload( + { + runs: list.runs, + hasAnyRuns: list.hasAnyRuns, + isLoading, + organizationSlug: organization.slug, + projectSlug: project.slug, + environmentSlug: environment.slug, + taskSlug, + } + ); + + const onClickShowNewRuns = () => { + const isPaginated = has("cursor") || has("direction"); + dismissNewRuns(); + if (isPaginated) { + replace({ cursor: undefined, direction: undefined }); + return; + } + revalidator.revalidate(); + }; + + // Surface the banner to the top-bar button rendered by the page: keep the + // ref's action current, mirror the count up, and clear it when this boundary + // unmounts (e.g. the table re-suspends on a filter change). + useEffect(() => { + showNewRunsRef.current = onClickShowNewRuns; + }, [onClickShowNewRuns, showNewRunsRef]); + useEffect(() => { + onNewRunsCountChange(newRunsCount); + }, [newRunsCount, onNewRunsCountChange]); + useEffect(() => () => onNewRunsCountChange(0), [onNewRunsCountChange]); + + return ( +
+ +
+ ); +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/useRunsLiveReload.ts b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/useRunsLiveReload.ts index a8d26ac350b..e443fb84cfb 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/useRunsLiveReload.ts +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/useRunsLiveReload.ts @@ -48,12 +48,22 @@ function isNewRunsCheckTick(tick: number) { function appendNewRunsSearchParams( searchParams: URLSearchParams, - { locationSearch, since }: { locationSearch: string; since: number } + { locationSearch, since, taskSlug }: { locationSearch: string; since: number; taskSlug?: string } ) { const filterParams = filterParamsWithoutPagination(locationSearch); for (const [key, value] of filterParams) { searchParams.append(key, value); } + // On the task landing pages the task lives in the route path, not the query + // string, so scope the new-runs count to this task explicitly. The task pages + // list every run of the task (their loaders apply no rootOnly filter), so + // force rootOnly off for the count too: a rootOnly preference persisted from + // the main Runs page would otherwise make the count skip child runs the list + // is showing, so the "N new runs" button could under-count or never appear. + if (taskSlug) { + searchParams.append("tasks", taskSlug); + searchParams.set("rootOnly", "false"); + } searchParams.set("includeNewRuns", "true"); searchParams.set("since", String(since)); } @@ -138,6 +148,7 @@ export function useRunsLiveReload({ organizationSlug, projectSlug, environmentSlug, + taskSlug, }: { runs: ListedRun[]; hasAnyRuns: boolean; @@ -145,6 +156,11 @@ export function useRunsLiveReload({ organizationSlug: string; projectSlug: string; environmentSlug: string; + /** + * When set, scopes new-run detection to this task. Used by the task landing + * pages, where the task is a route path param rather than a `tasks` filter. + */ + taskSlug?: string; }) { const location = useLocation(); const runsPollFetcher = useTypedFetcher(); @@ -230,6 +246,7 @@ export function useRunsLiveReload({ appendNewRunsSearchParams(searchParams, { locationSearch: location.search, since: knownNewestRunMs, + taskSlug, }); } @@ -242,6 +259,7 @@ export function useRunsLiveReload({ knownNewestRunMs, runsPollFetcher, runsResourcesBasePath, + taskSlug, ] ); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx index 9ef03b5c189..398c6135531 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx @@ -56,7 +56,6 @@ import { EnabledStatus } from "~/components/runs/v3/EnabledStatus"; import type { TaskRunListSearchFilters } from "~/components/runs/v3/RunFilters"; import { ScheduleTypeIcon, scheduleTypeName } from "~/components/runs/v3/ScheduleType"; import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; -import { TaskRunsTable } from "~/components/runs/v3/TaskRunsTable"; import { ScheduleInspector } from "~/components/schedules/ScheduleInspector"; import { ScheduleLimitActions } from "~/components/schedules/ScheduleLimitActions"; import { SchedulesUsageBar } from "~/components/schedules/SchedulesUsageBar"; @@ -95,6 +94,7 @@ import type { loader as scheduleEditLoader } from "../_app.orgs.$organizationSlu import type { loader as scheduleNewLoader } from "../_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.new/route"; import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; import { UpsertScheduleForm } from "../resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.new/route"; +import { NewRunsButton, TaskRunsList } from "~/components/runs/v3/TaskRunsList"; export const meta: MetaFunction = ({ data }) => { const slug = (data as { task?: TaskDetail | null } | undefined)?.task?.slug; @@ -181,6 +181,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { to, cursor, direction, + includeHasAnyRuns: true, }) .catch(() => null); @@ -231,6 +232,13 @@ export default function Page() { !!plan?.v3Subscription?.plan && !plan.v3Subscription.plan.limits.schedules.canExceed; const isAtLimit = !!limits && limits.used >= limits.limit; + // New-runs banner state is lifted here so the button can live in the top bar, + // while the count/action originate from the live-reload hook inside the + // deferred runs table below. Count drives visibility; the ref exposes the + // click action (kept current by TaskRunsList each render). + const [newRunsCount, setNewRunsCount] = useState(0); + const showNewRunsRef = useRef<() => void>(() => {}); + return ( @@ -265,6 +273,9 @@ export default function Page() { onCreate={openCreateSchedule} disabled={isCreatingSchedule} /> + {newRunsCount > 0 ? ( + showNewRunsRef.current()} /> + ) : null} }> {(list) => list ? ( -
- -
+ ) : ( ) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx index f38124f2596..ac765406ce1 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx @@ -1,7 +1,7 @@ import { type MetaFunction } from "@remix-run/react"; import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { formatDurationMilliseconds } from "@trigger.dev/core/v3"; -import { Suspense, useMemo } from "react"; +import { Suspense, useMemo, useRef, useState } from "react"; import { TypedAwait, typeddefer, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; import { BeakerIcon } from "~/assets/icons/BeakerIcon"; @@ -29,7 +29,6 @@ import { } from "~/components/primitives/Resizable"; import { Spinner } from "~/components/primitives/Spinner"; import { TextLink } from "~/components/primitives/TextLink"; -import { TaskRunsTable } from "~/components/runs/v3/TaskRunsTable"; import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; import { $replica } from "~/db.server"; import { useEnvironment } from "~/hooks/useEnvironment"; @@ -52,6 +51,7 @@ import { v3TestTaskPath, } from "~/utils/pathBuilder"; import { parseFiniteInt } from "~/utils/searchParams"; +import { NewRunsButton, TaskRunsList } from "~/components/runs/v3/TaskRunsList"; export const meta: MetaFunction = ({ data }) => { const slug = (data as { task?: TaskDetail | null } | undefined)?.task?.slug; @@ -121,6 +121,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { to, cursor, direction, + includeHasAnyRuns: true, }) .catch(() => null); @@ -144,6 +145,13 @@ export default function Page() { }); const queuesPath = v3QueuesPath(organization, project, environment); + // New-runs banner state is lifted here so the button can live in the top bar, + // while the count/action originate from the live-reload hook inside the + // deferred runs table below. Count drives visibility; the ref exposes the + // click action (kept current by TaskRunsList each render). + const [newRunsCount, setNewRunsCount] = useState(0); + const showNewRunsRef = useRef<() => void>(() => {}); + return ( @@ -166,6 +174,9 @@ export default function Page() {
Runs
+ {newRunsCount > 0 ? ( + showNewRunsRef.current()} /> + ) : null} @@ -200,17 +211,12 @@ export default function Page() { }> {(list) => list ? ( -
- -
+ ) : ( ) From efd0ee8d74a2548e125d22ba801647a1b0f458cd Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 27 Jul 2026 16:19:27 +0100 Subject: [PATCH 16/67] feat(core,sdk): support additional environment API keys (#4387) ## Summary Additional environment API keys can use SDK APIs that require public access tokens. The SDK detects the additional-key format and asks the Trigger.dev server to mint scoped tokens instead of attempting to sign them locally. Root environment keys retain their existing local-signing behavior. Trigger and batch clients also prefer server-issued tokens returned in response headers while preserving compatibility with older servers. ## Deployment notes This package update is safe to publish before servers expose additional key creation. Existing root keys continue to use the current path, while an additional key used with an older server fails with an actionable upgrade error. --- .changeset/friendly-keys-batch-tokens.md | 6 + packages/core/src/v3/apiClient/index.ts | 67 +++++- packages/core/src/v3/apiKeys.test.ts | 18 ++ packages/core/src/v3/apiKeys.ts | 11 + packages/core/src/v3/index.ts | 1 + packages/trigger-sdk/src/v3/ai.ts | 38 +++- packages/trigger-sdk/src/v3/auth.test.ts | 200 ++++++++++++++++++ packages/trigger-sdk/src/v3/auth.ts | 109 ++++++++-- .../src/v3/createStartSessionAction.test.ts | 38 ++++ 9 files changed, 464 insertions(+), 24 deletions(-) create mode 100644 .changeset/friendly-keys-batch-tokens.md create mode 100644 packages/core/src/v3/apiKeys.test.ts create mode 100644 packages/core/src/v3/apiKeys.ts create mode 100644 packages/trigger-sdk/src/v3/auth.test.ts diff --git a/.changeset/friendly-keys-batch-tokens.md b/.changeset/friendly-keys-batch-tokens.md new file mode 100644 index 00000000000..505edacd334 --- /dev/null +++ b/.changeset/friendly-keys-batch-tokens.md @@ -0,0 +1,6 @@ +--- +"@trigger.dev/core": patch +"@trigger.dev/sdk": patch +--- + +Allow additional environment API keys to create scoped public access tokens through the Trigger.dev API. Use server-issued public access tokens for batch operations so environment-scoped API keys can read batch results. diff --git a/packages/core/src/v3/apiClient/index.ts b/packages/core/src/v3/apiClient/index.ts index deda5d01f9a..5329f15fad0 100644 --- a/packages/core/src/v3/apiClient/index.ts +++ b/packages/core/src/v3/apiClient/index.ts @@ -1,6 +1,7 @@ import { nanoid } from "nanoid"; import { z } from "zod"; import { VERSION } from "../../version.js"; +import { isAdditionalApiKey } from "../apiKeys.js"; import type { ApiClientConfiguration } from "../apiClientManager-api.js"; import { generateJWT } from "../jwt.js"; import { @@ -201,6 +202,15 @@ export type { export * from "./getBranch.js"; +export type CreatePublicTokenRequestBody = { + scopes: string[]; + expirationTime?: string | number; + oneTimeUse?: boolean; + realtime?: { skipColumns?: string[] }; +}; + +const CreatePublicTokenResponseBody = z.object({ token: z.string() }); + /** * Trigger.dev v3 API client */ @@ -230,6 +240,21 @@ export class ApiClient { this.futureFlags = futureFlags; } + /** + * Key for signing a public access token locally. Only root keys can do this — + * an additional key isn't the environment's signing material, so a token + * signed with one would never verify. Throw rather than return a dead token. + */ + get #selfSigningKey(): string { + if (isAdditionalApiKey(this.accessToken)) { + throw new Error( + "This additional API key cannot self-sign public tokens, and the server did not return one. Upgrade the server or use the root API key." + ); + } + + return this.accessToken; + } + get fetchClient(): typeof fetch { const headers = this.#getHeaders(false); @@ -325,7 +350,7 @@ export class ApiClient { const claims = claimsHeader ? JSON.parse(claimsHeader) : undefined; const jwt = await generateJWT({ - secretKey: this.accessToken, + secretKey: this.#selfSigningKey, payload: { ...claims, scopes: [`read:runs:${data.id}`], @@ -359,11 +384,20 @@ export class ApiClient { ) .withResponse() .then(async ({ data, response }) => { + const jwtHeader = response.headers.get("x-trigger-jwt"); + + if (typeof jwtHeader === "string") { + return { + ...data, + publicAccessToken: jwtHeader, + }; + } + const claimsHeader = response.headers.get("x-trigger-jwt-claims"); const claims = claimsHeader ? JSON.parse(claimsHeader) : undefined; const jwt = await generateJWT({ - secretKey: this.accessToken, + secretKey: this.#selfSigningKey, payload: { ...claims, scopes: [`read:batch:${data.id}`], @@ -407,11 +441,20 @@ export class ApiClient { ) .withResponse() .then(async ({ data, response }) => { + const jwtHeader = response.headers.get("x-trigger-jwt"); + + if (typeof jwtHeader === "string") { + return { + ...data, + publicAccessToken: jwtHeader, + }; + } + const claimsHeader = response.headers.get("x-trigger-jwt-claims"); const claims = claimsHeader ? JSON.parse(claimsHeader) : undefined; const jwt = await generateJWT({ - secretKey: this.accessToken, + secretKey: this.#selfSigningKey, payload: { ...claims, scopes: [`read:batch:${data.id}`], @@ -1107,7 +1150,7 @@ export class ApiClient { const claims = claimsHeader ? JSON.parse(claimsHeader) : undefined; const jwt = await generateJWT({ - secretKey: this.accessToken, + secretKey: this.#selfSigningKey, payload: { ...claims, scopes: [`write:waitpoints:${data.id}`], @@ -1870,6 +1913,22 @@ export class ApiClient { ); } + async createPublicToken( + body: CreatePublicTokenRequestBody, + requestOptions?: ZodFetchOptions + ): Promise<{ token: string }> { + return zodfetch( + CreatePublicTokenResponseBody, + `${this.baseUrl}/api/v1/auth/public-tokens`, + { + method: "POST", + headers: this.#getHeaders(false), + body: JSON.stringify(body), + }, + mergeRequestOptions(this.defaultRequestOptions, requestOptions) + ); + } + retrieveBatch(batchId: string, requestOptions?: ZodFetchOptions) { return zodfetch( RetrieveBatchV2Response, diff --git a/packages/core/src/v3/apiKeys.test.ts b/packages/core/src/v3/apiKeys.test.ts new file mode 100644 index 00000000000..8d0108fb633 --- /dev/null +++ b/packages/core/src/v3/apiKeys.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; +import { isAdditionalApiKey } from "./apiKeys.js"; + +describe("isAdditionalApiKey", () => { + it.each(["dev", "stg", "prod", "preview"])("recognizes %s additional keys", (environment) => { + expect(isAdditionalApiKey(`tr_${environment}_sk_0123456789abcdefghijklmn`)).toBe(true); + }); + + it.each([ + "tr_prod_0123456789abcdefghijklmn", + "tr_prod_sk_too-short", + "tr_prod_sk_0123456789abcdefghijklmn_extra", + "tr_test_sk_0123456789abcdefghijklmn", + "tr_prod_sk_0123456789abcdefghijkl_", + ])("rejects %s", (key) => { + expect(isAdditionalApiKey(key)).toBe(false); + }); +}); diff --git a/packages/core/src/v3/apiKeys.ts b/packages/core/src/v3/apiKeys.ts new file mode 100644 index 00000000000..2571d665030 --- /dev/null +++ b/packages/core/src/v3/apiKeys.ts @@ -0,0 +1,11 @@ +const ADDITIONAL_API_KEY_PATTERN = /^tr_(dev|stg|prod|preview)_sk_[0-9a-zA-Z]{24}$/; + +/** + * Returns whether a key has the additional environment API key format. + * + * This is only a routing hint. It must never be used as a security boundary; + * servers authenticate additional keys by resolving their stored hash. + */ +export function isAdditionalApiKey(key: string): boolean { + return ADDITIONAL_API_KEY_PATTERN.test(key); +} diff --git a/packages/core/src/v3/index.ts b/packages/core/src/v3/index.ts index 35dbfa287f9..10551fca6f6 100644 --- a/packages/core/src/v3/index.ts +++ b/packages/core/src/v3/index.ts @@ -30,6 +30,7 @@ export * from "./resource-catalog-api.js"; export * from "./types/index.js"; export { links } from "./links.js"; export * from "./jwt.js"; +export * from "./apiKeys.js"; export * from "./workloadDeploymentToken.js"; export * from "./idempotencyKeys.js"; export * from "./streams/asyncIterableStream.js"; diff --git a/packages/trigger-sdk/src/v3/ai.ts b/packages/trigger-sdk/src/v3/ai.ts index 0931ff1d89e..65ec72f4308 100644 --- a/packages/trigger-sdk/src/v3/ai.ts +++ b/packages/trigger-sdk/src/v3/ai.ts @@ -14,6 +14,7 @@ import { type inferSchemaOut, InputStreamOncePromise, type InputStreamOnceResult, + isAdditionalApiKey, isSchemaZodEsque, logger, type MachinePresetName, @@ -10600,24 +10601,53 @@ async function mintPublicTokenWithOverride(args: { throw new Error("chat.createStartSessionAction: no API access token configured for JWT mint."); } const ctx: ChatStartSessionEndpointContext = { endpoint: "auth", chatId: args.chatId }; - const url = `${resolveChatStartBaseURL("auth", args.chatId, args.baseURLOption)}/api/v1/auth/jwt/claims`; + const scopes = [`read:sessions:${args.chatId}`, `write:sessions:${args.chatId}`]; + const serverMint = isAdditionalApiKey(accessToken); + const endpoint = serverMint ? "/api/v1/auth/public-tokens" : "/api/v1/auth/jwt/claims"; + const url = `${resolveChatStartBaseURL("auth", args.chatId, args.baseURLOption)}${endpoint}`; const init: RequestInit = { method: "POST", headers: overrideRequestHeaders(accessToken), + ...(serverMint + ? { + body: JSON.stringify({ + scopes, + expirationTime: + args.expirationTime instanceof Date + ? Math.floor(args.expirationTime.getTime() / 1000) + : args.expirationTime, + }), + } + : {}), }; const response = args.fetchOverride ? await args.fetchOverride(url, init, ctx) : await fetch(url, init); if (!response.ok) { + // An additional API key cannot self-sign, so it must use the server mint + // endpoint. On a server too old to expose it, explain the recovery path + // instead of surfacing a bare 404 (mirrors auth.createServerPublicToken). + if (serverMint && response.status === 404) { + throw new Error( + "This additional API key cannot self-sign public tokens, and the server does not support public-token minting. Upgrade the server or use the root API key." + ); + } const text = await response.text().catch(() => ""); throw new Error(`auth.createPublicToken failed: ${response.status} ${text}`); } - const claims = (await response.json()) as Record; + const responseBody = (await response.json()) as Record; + if (serverMint) { + if (typeof responseBody.token !== "string") { + throw new Error("auth.createPublicToken failed: server response did not include a token"); + } + return responseBody.token; + } + return generateJWT({ secretKey: accessToken, payload: { - ...claims, - scopes: [`read:sessions:${args.chatId}`, `write:sessions:${args.chatId}`], + ...responseBody, + scopes, }, expirationTime: args.expirationTime, }); diff --git a/packages/trigger-sdk/src/v3/auth.test.ts b/packages/trigger-sdk/src/v3/auth.test.ts new file mode 100644 index 00000000000..c32ce788a53 --- /dev/null +++ b/packages/trigger-sdk/src/v3/auth.test.ts @@ -0,0 +1,200 @@ +import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { apiClientManager } from "@trigger.dev/core/v3"; +import { validateJWT } from "@trigger.dev/core/v3/jwt"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { auth } from "./auth.js"; + +type ReceivedRequest = { + method: string; + url: string; + authorization?: string; + body: unknown; +}; + +describe("public token API key routing", () => { + let server: Server; + let baseUrl: string; + let requests: ReceivedRequest[]; + let publicTokenStatus: number; + + beforeEach(async () => { + requests = []; + publicTokenStatus = 200; + server = createServer((request, response) => { + void handleRequest(request, response, requests, () => publicTokenStatus); + }); + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const address = server.address() as AddressInfo; + baseUrl = `http://127.0.0.1:${address.port}`; + resolve(); + }); + }); + }); + + afterEach(async () => { + await new Promise((resolve) => server.close(() => resolve())); + }); + + it("uses server minting for additional keys", async () => { + const key = "tr_prod_sk_0123456789abcdefghijklmn"; + const token = await apiClientManager.runWithConfig({ baseURL: baseUrl, accessToken: key }, () => + auth.createPublicToken({ + scopes: { read: { runs: ["run_123"] } }, + expirationTime: new Date("2030-01-01T00:00:00.000Z"), + realtime: { skipColumns: ["payload"] }, + }) + ); + + expect(token).toBe("server-minted-token"); + expect(requests).toEqual([ + { + method: "POST", + url: "/api/v1/auth/public-tokens", + authorization: `Bearer ${key}`, + body: { + scopes: ["read:runs:run_123"], + expirationTime: 1893456000, + realtime: { skipColumns: ["payload"] }, + }, + }, + ]); + }); + + it("server-mints trigger tokens with one-time-use semantics", async () => { + const key = "tr_dev_sk_0123456789abcdefghijklmn"; + await apiClientManager.runWithConfig({ baseURL: baseUrl, accessToken: key }, () => + auth.createTriggerPublicToken(["task-one", "task-two"], { multipleUse: true }) + ); + + expect(requests[0]?.body).toEqual({ + scopes: ["trigger:tasks:task-one", "trigger:tasks:task-two"], + oneTimeUse: false, + }); + }); + + it("server-mints batch trigger tokens for additional keys", async () => { + const key = "tr_stg_sk_0123456789abcdefghijklmn"; + await apiClientManager.runWithConfig({ baseURL: baseUrl, accessToken: key }, () => + auth.createBatchTriggerPublicToken("batch-task", { + expirationTime: "2h", + multipleUse: false, + realtime: { skipColumns: ["payload"] }, + }) + ); + + expect(requests).toEqual([ + { + method: "POST", + url: "/api/v1/auth/public-tokens", + authorization: `Bearer ${key}`, + body: { + scopes: ["batchTrigger:tasks:batch-task"], + expirationTime: "2h", + oneTimeUse: true, + realtime: { skipColumns: ["payload"] }, + }, + }, + ]); + }); + + it("keeps root key self-minting unchanged", async () => { + const key = "tr_prod_0123456789abcdefghijklmn"; + const token = await apiClientManager.runWithConfig({ baseURL: baseUrl, accessToken: key }, () => + auth.createPublicToken({ scopes: { read: { runs: true } } }) + ); + + expect(requests.map((request) => request.url)).toEqual(["/api/v1/auth/jwt/claims"]); + const validation = await validateJWT(token, key); + expect(validation.ok).toBe(true); + if (!validation.ok) return; + expect(validation.payload).toMatchObject({ + sub: "env_test", + pub: true, + scopes: ["read:runs"], + }); + }); + + it.each([ + ["no options at all", undefined], + ["an empty scopes object", {}], + ["a scope group with nothing selected", { read: {} }], + ])("keeps root-key behavior unchanged for %s", async (_label, scopes) => { + const key = "tr_prod_0123456789abcdefghijklmn"; + const token = await apiClientManager.runWithConfig({ baseURL: baseUrl, accessToken: key }, () => + auth.createPublicToken(scopes === undefined ? undefined : { scopes }) + ); + + expect(requests.map((request) => request.url)).toEqual(["/api/v1/auth/jwt/claims"]); + const validation = await validateJWT(token, key); + expect(validation.ok).toBe(true); + }); + + it.each([ + ["no options at all", undefined], + ["an empty scopes object", {}], + ["a scope group with nothing selected", { read: {} }], + ])("rejects %s clearly for additional keys", async (_label, scopes) => { + const key = "tr_prod_sk_0123456789abcdefghijklmn"; + const promise = apiClientManager.runWithConfig({ baseURL: baseUrl, accessToken: key }, () => + auth.createPublicToken(scopes === undefined ? undefined : { scopes }) + ); + + await expect(promise).rejects.toThrow( + "requires at least one scope when using an additional API key" + ); + expect(requests).toEqual([]); + }); + + it("explains how to recover when the server lacks the mint endpoint", async () => { + publicTokenStatus = 404; + const promise = apiClientManager.runWithConfig( + { + baseURL: baseUrl, + accessToken: "tr_prod_sk_0123456789abcdefghijklmn", + }, + () => auth.createPublicToken({ scopes: { read: { runs: true } } }) + ); + + await expect(promise).rejects.toThrow("Upgrade the server or use the root API key"); + }); +}); + +async function handleRequest( + request: IncomingMessage, + response: ServerResponse, + requests: ReceivedRequest[], + publicTokenStatus: () => number +) { + const chunks: Buffer[] = []; + for await (const chunk of request) { + chunks.push(Buffer.from(chunk)); + } + const rawBody = Buffer.concat(chunks).toString(); + requests.push({ + method: request.method ?? "", + url: request.url ?? "", + authorization: request.headers.authorization, + body: rawBody ? JSON.parse(rawBody) : undefined, + }); + + if (request.url === "/api/v1/auth/jwt/claims") { + return json(response, { sub: "env_test", pub: true }); + } + if (request.url === "/api/v1/auth/public-tokens") { + const status = publicTokenStatus(); + return json( + response, + status === 200 ? { token: "server-minted-token" } : { error: "Not found" }, + status + ); + } + + return json(response, { error: "Not found" }, 404); +} + +function json(response: ServerResponse, body: unknown, status = 200) { + response.writeHead(status, { "content-type": "application/json" }); + response.end(JSON.stringify(body)); +} diff --git a/packages/trigger-sdk/src/v3/auth.ts b/packages/trigger-sdk/src/v3/auth.ts index 5e76765956b..d26a08fa874 100644 --- a/packages/trigger-sdk/src/v3/auth.ts +++ b/packages/trigger-sdk/src/v3/auth.ts @@ -1,8 +1,11 @@ import { type RealtimeRunSkipColumns, + type ApiClient, type ApiClientConfiguration, + type CreatePublicTokenRequestBody, apiClientManager, generateJWT as internal_generateJWT, + isAdditionalApiKey, } from "@trigger.dev/core/v3"; import "@trigger.dev/core/v3/sdk-scope-storage"; @@ -102,7 +105,9 @@ export type PublicTokenPermissions = { export type CreatePublicTokenOptions = { /** - * A collection of permission scopes to be granted to the token. + * A collection of permission scopes to be granted to the token. This remains + * optional for root API key compatibility; additional API keys require at + * least one scope. * * @example * @@ -117,7 +122,8 @@ export type CreatePublicTokenOptions = { scopes?: PublicTokenPermissions; /** - * The expiration time for the token. This can be a number representing the time in milliseconds, a `Date` object, or a string. + * The expiration time for the token: a duration string, a `Date`, or a Unix + * timestamp in **seconds**. * * @example * @@ -146,6 +152,35 @@ export type CreatePublicTokenOptions = { }; }; +// A `Date` becomes Unix seconds; duration strings and numbers pass through. A +// bare number is already Unix seconds, matching how the local signing path +// reads it, so both key types treat the option the same. +function serverExpirationTime( + expirationTime: number | Date | string | undefined +): number | string | undefined { + return expirationTime instanceof Date + ? Math.floor(expirationTime.getTime() / 1000) + : expirationTime; +} + +async function createServerPublicToken( + apiClient: ApiClient, + body: CreatePublicTokenRequestBody +): Promise { + try { + const result = await apiClient.createPublicToken(body); + return result.token; + } catch (error) { + if (error !== null && typeof error === "object" && "status" in error && error.status === 404) { + throw new Error( + "This additional API key cannot self-sign public tokens, and the server does not support public-token minting. Upgrade the server or use the root API key." + ); + } + + throw error; + } +} + /** * Creates a public token using the provided options. * @@ -170,15 +205,34 @@ export type CreatePublicTokenOptions = { * ``` */ async function createPublicToken(options?: CreatePublicTokenOptions): Promise { + const scopes = options?.scopes ? flattenScopes(options.scopes) : []; const apiClient = apiClientManager.clientOrThrow(); + if (isAdditionalApiKey(apiClient.accessToken)) { + // Additional keys cannot self-sign, and the server rejects empty scope + // lists because they cannot authorize anything. Keep the legacy root-key + // behaviour unchanged while failing clearly for this new credential type. + if (scopes.length === 0) { + throw new Error( + "auth.createPublicToken() requires at least one scope when using an additional API key. " + + 'For example: auth.createPublicToken({ scopes: { read: { runs: ["run_1234"] } } })' + ); + } + + return createServerPublicToken(apiClient, { + scopes, + expirationTime: serverExpirationTime(options?.expirationTime), + realtime: options?.realtime, + }); + } + const claims = await apiClient.generateJWTClaims(); return await internal_generateJWT({ secretKey: apiClient.accessToken, payload: { ...claims, - scopes: options?.scopes ? flattenScopes(options.scopes) : undefined, + scopes: options?.scopes ? scopes : undefined, realtime: options?.realtime, }, expirationTime: options?.expirationTime, @@ -199,7 +253,8 @@ async function withPublicToken(options: CreatePublicTokenOptions, fn: () => Prom export type CreateTriggerTokenOptions = { /** - * The expiration time for the token. This can be a number representing the time in milliseconds, a `Date` object, or a string. + * The expiration time for the token: a duration string, a `Date`, or a Unix + * timestamp in **seconds**. * * @example * @@ -271,6 +326,21 @@ async function createTriggerPublicToken( options?: CreateTriggerTokenOptions ): Promise { const apiClient = apiClientManager.clientOrThrow(); + const scopes = flattenScopes({ + trigger: { + tasks: task, + }, + }); + const oneTimeUse = typeof options?.multipleUse === "boolean" ? !options.multipleUse : true; + + if (isAdditionalApiKey(apiClient.accessToken)) { + return createServerPublicToken(apiClient, { + scopes, + expirationTime: serverExpirationTime(options?.expirationTime), + oneTimeUse, + realtime: options?.realtime, + }); + } const claims = await apiClient.generateJWTClaims(); @@ -278,13 +348,9 @@ async function createTriggerPublicToken( secretKey: apiClient.accessToken, payload: { ...claims, - otu: typeof options?.multipleUse === "boolean" ? !options.multipleUse : true, + otu: oneTimeUse, realtime: options?.realtime, - scopes: flattenScopes({ - trigger: { - tasks: task, - }, - }), + scopes, }, expirationTime: options?.expirationTime, }); @@ -343,6 +409,21 @@ async function createBatchTriggerPublicToken( options?: CreateTriggerTokenOptions ): Promise { const apiClient = apiClientManager.clientOrThrow(); + const scopes = flattenScopes({ + batchTrigger: { + tasks: task, + }, + }); + const oneTimeUse = typeof options?.multipleUse === "boolean" ? !options.multipleUse : true; + + if (isAdditionalApiKey(apiClient.accessToken)) { + return createServerPublicToken(apiClient, { + scopes, + expirationTime: serverExpirationTime(options?.expirationTime), + oneTimeUse, + realtime: options?.realtime, + }); + } const claims = await apiClient.generateJWTClaims(); @@ -350,13 +431,9 @@ async function createBatchTriggerPublicToken( secretKey: apiClient.accessToken, payload: { ...claims, - otu: typeof options?.multipleUse === "boolean" ? !options.multipleUse : true, + otu: oneTimeUse, realtime: options?.realtime, - scopes: flattenScopes({ - batchTrigger: { - tasks: task, - }, - }), + scopes, }, expirationTime: options?.expirationTime, }); diff --git a/packages/trigger-sdk/src/v3/createStartSessionAction.test.ts b/packages/trigger-sdk/src/v3/createStartSessionAction.test.ts index d1a6a9b61d2..ca18ce59985 100644 --- a/packages/trigger-sdk/src/v3/createStartSessionAction.test.ts +++ b/packages/trigger-sdk/src/v3/createStartSessionAction.test.ts @@ -132,6 +132,44 @@ describe("chat.createStartSessionAction — runtime", () => { expect(lastStartBody?.triggerConfig.lockToVersion).toBe("20260101.1"); }); + it("server-mints override tokens for additional API keys", async () => { + const requests: Array<{ url: string; body: unknown }> = []; + const start = chat.createStartSessionAction("fake-chat", { + apiClient: { + baseURL: "https://example.invalid", + accessToken: "tr_prod_sk_0123456789abcdefghijklmn", + }, + tokenTTL: "30m", + fetch: async (url, init, context) => { + requests.push({ + url, + body: typeof init.body === "string" ? JSON.parse(init.body) : undefined, + }); + + if (context.endpoint === "sessions") { + return Response.json({ + id: "session_fixture", + runId: "run_fixture", + publicAccessToken: "session-token", + }); + } + + return Response.json({ token: "server-minted-token" }); + }, + }); + + const result = await start({ chatId: "chat-additional-key" }); + + expect(result.publicAccessToken).toBe("server-minted-token"); + expect(requests[1]).toEqual({ + url: "https://example.invalid/api/v1/auth/public-tokens", + body: { + scopes: ["read:sessions:chat-additional-key", "write:sessions:chat-additional-key"], + expirationTime: "30m", + }, + }); + }); + it("keeps session-level metadata distinct from per-turn clientData", async () => { installStartFixture(); From d30ee6e5700ecc7eaaff0c92bc05d1132573b3a8 Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Mon, 27 Jul 2026 16:29:36 +0100 Subject: [PATCH 17/67] feat(webapp): favorite pages and sidebar customization (#4375) ## Summary Favorite any dashboard page and it appears in a new "Favorites" section at the top of the side menu. The star next to the page title (or Option+F) saves the exact view, filters and tabs included, with a name derived from the URL ("Runs: Completed successfully, last 7d", "Run: 05hrqq9n") that you can rename inline from each item's hover menu. The sidebar is customizable too: "Customize sidebar" (on section header menus and in each "More" menu) opens a modal where you can reorder sections, drag items into a new order, hide items behind a per-section "More" popover, and rename or remove favorites. Changes apply on Confirm, Reset restores the default layout without touching favorites, and everything is stored per user in dashboard preferences. ## Screenshots | Favorites in the side menu | Customize sidebar modal | | --- | --- | | ![Favorites section with rename and remove menu](https://raw.githubusercontent.com/triggerdotdev/trigger.dev/d56f073dc517e2073b01d8eff880183539638f03/favorites-side-menu.png) | ![Customize sidebar modal](https://raw.githubusercontent.com/triggerdotdev/trigger.dev/d56f073dc517e2073b01d8eff880183539638f03/customize-sidebar-modal.png) | ![Favorite star and tooltip in the page header](https://raw.githubusercontent.com/triggerdotdev/trigger.dev/d56f073dc517e2073b01d8eff880183539638f03/star-tooltip.png) ## Design notes - Favorite links carry a small marker search param so the favorite, not its identical main menu item, highlights as active. Markers from shared or stale links are cleaned on load, and changing any filter hands the highlight back to the regular menu item. - Preference writes are serialized with a row lock: several writers (debounced collapse and width saves, favorite toggles, the customize modal) can land concurrently and would otherwise clobber each other's read-modify-write of the JSON column. - Option+F is matched on `event.code` with a raw listener because macOS reports Option-modified letters as symbols, which the `event.key` based shortcut hook can't capture. Verified end-to-end in the browser: star toggle and shortcut, instant section appearance, inline rename and staged modal removal, filter-aware labels and unique active states, shared-link normalization, drag reordering, and persistence across reloads. --- ...avorite-pages-and-sidebar-customization.md | 6 + apps/webapp/app/assets/icons/CrossIcon.tsx | 14 + .../webapp/app/assets/icons/EyeClosedIcon.tsx | 35 + apps/webapp/app/assets/icons/EyeOpenIcon.tsx | 27 + apps/webapp/app/assets/icons/RenameIcon.tsx | 27 + .../app/assets/icons/SidebarCustomizeIcon.tsx | 26 + apps/webapp/app/components/Shortcuts.tsx | 4 + .../navigation/CustomizeSidebarDialog.tsx | 506 ++++++++++ .../components/navigation/DashboardList.tsx | 1 + .../navigation/FavoritePageButton.tsx | 144 +++ .../navigation/FavoritesSection.tsx | 198 ++++ .../app/components/navigation/SideMenu.tsx | 921 +++++++++++++----- .../components/navigation/SideMenuItem.tsx | 85 +- .../components/navigation/SideMenuSection.tsx | 29 +- .../components/navigation/favoritePages.tsx | 511 ++++++++++ .../components/navigation/sideMenuTypes.ts | 56 ++ .../app/components/primitives/PageHeader.tsx | 34 +- .../route.tsx | 13 +- .../resources.preferences.favorites.tsx | 80 ++ .../routes/resources.preferences.sidemenu.tsx | 64 +- .../services/dashboardPreferences.server.ts | 415 ++++++-- 21 files changed, 2838 insertions(+), 358 deletions(-) create mode 100644 .server-changes/favorite-pages-and-sidebar-customization.md create mode 100644 apps/webapp/app/assets/icons/CrossIcon.tsx create mode 100644 apps/webapp/app/assets/icons/EyeClosedIcon.tsx create mode 100644 apps/webapp/app/assets/icons/EyeOpenIcon.tsx create mode 100644 apps/webapp/app/assets/icons/RenameIcon.tsx create mode 100644 apps/webapp/app/assets/icons/SidebarCustomizeIcon.tsx create mode 100644 apps/webapp/app/components/navigation/CustomizeSidebarDialog.tsx create mode 100644 apps/webapp/app/components/navigation/FavoritePageButton.tsx create mode 100644 apps/webapp/app/components/navigation/FavoritesSection.tsx create mode 100644 apps/webapp/app/components/navigation/favoritePages.tsx create mode 100644 apps/webapp/app/routes/resources.preferences.favorites.tsx diff --git a/.server-changes/favorite-pages-and-sidebar-customization.md b/.server-changes/favorite-pages-and-sidebar-customization.md new file mode 100644 index 00000000000..d4468b4df51 --- /dev/null +++ b/.server-changes/favorite-pages-and-sidebar-customization.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: feature +--- + +Favorite any dashboard page to a new Favorites section in the side menu, and customize the sidebar by renaming favorites, hiding items, and reordering items and sections. diff --git a/apps/webapp/app/assets/icons/CrossIcon.tsx b/apps/webapp/app/assets/icons/CrossIcon.tsx new file mode 100644 index 00000000000..fa0c714602f --- /dev/null +++ b/apps/webapp/app/assets/icons/CrossIcon.tsx @@ -0,0 +1,14 @@ +export function CrossIcon({ className }: { className?: string }) { + return ( + + + + ); +} diff --git a/apps/webapp/app/assets/icons/EyeClosedIcon.tsx b/apps/webapp/app/assets/icons/EyeClosedIcon.tsx new file mode 100644 index 00000000000..c83d2499267 --- /dev/null +++ b/apps/webapp/app/assets/icons/EyeClosedIcon.tsx @@ -0,0 +1,35 @@ +export function EyeClosedIcon({ className }: { className?: string }) { + return ( + + + + + + + ); +} diff --git a/apps/webapp/app/assets/icons/EyeOpenIcon.tsx b/apps/webapp/app/assets/icons/EyeOpenIcon.tsx new file mode 100644 index 00000000000..e00cfa0fcf5 --- /dev/null +++ b/apps/webapp/app/assets/icons/EyeOpenIcon.tsx @@ -0,0 +1,27 @@ +export function EyeOpenIcon({ className }: { className?: string }) { + return ( + + + + + ); +} diff --git a/apps/webapp/app/assets/icons/RenameIcon.tsx b/apps/webapp/app/assets/icons/RenameIcon.tsx new file mode 100644 index 00000000000..6514e0b09cd --- /dev/null +++ b/apps/webapp/app/assets/icons/RenameIcon.tsx @@ -0,0 +1,27 @@ +export function RenameIcon({ className }: { className?: string }) { + return ( + + + + + ); +} diff --git a/apps/webapp/app/assets/icons/SidebarCustomizeIcon.tsx b/apps/webapp/app/assets/icons/SidebarCustomizeIcon.tsx new file mode 100644 index 00000000000..0960e4a2011 --- /dev/null +++ b/apps/webapp/app/assets/icons/SidebarCustomizeIcon.tsx @@ -0,0 +1,26 @@ +export function SidebarCustomizeIcon({ className }: { className?: string }) { + return ( + + + + + ); +} diff --git a/apps/webapp/app/components/Shortcuts.tsx b/apps/webapp/app/components/Shortcuts.tsx index 87e5d08f371..c4ce2db6d0f 100644 --- a/apps/webapp/app/components/Shortcuts.tsx +++ b/apps/webapp/app/components/Shortcuts.tsx @@ -73,6 +73,10 @@ function ShortcutContent() { + + + + diff --git a/apps/webapp/app/components/navigation/CustomizeSidebarDialog.tsx b/apps/webapp/app/components/navigation/CustomizeSidebarDialog.tsx new file mode 100644 index 00000000000..d2c18442e3b --- /dev/null +++ b/apps/webapp/app/components/navigation/CustomizeSidebarDialog.tsx @@ -0,0 +1,506 @@ +import { DialogClose } from "@radix-ui/react-dialog"; +import { ArrowDownIcon, ArrowUpIcon } from "@heroicons/react/20/solid"; +import { GripVerticalIcon } from "lucide-react"; +import { useState } from "react"; +import ReactGridLayout, { type Layout, useContainerWidth } from "react-grid-layout"; +import { CrossIcon } from "~/assets/icons/CrossIcon"; +import { EyeClosedIcon } from "~/assets/icons/EyeClosedIcon"; +import { EyeOpenIcon } from "~/assets/icons/EyeOpenIcon"; +import { cn } from "~/utils/cn"; +import { Button } from "../primitives/Buttons"; +import { DialogContent, DialogFooter, DialogHeader } from "../primitives/Dialog"; +import { FormError } from "../primitives/FormError"; +import { Header3 } from "../primitives/Headers"; +import { Icon, type RenderIcon } from "../primitives/Icon"; +import { Input } from "../primitives/Input"; +import { isItemHidden, orderByPreference } from "./sideMenuTypes"; + +export type CustomizeSidebarItem = { + id: string; + name: string; + icon: RenderIcon; + iconClassName?: string; + defaultHidden?: boolean; + /** Favorites get an inline-editable name in the modal. */ + isFavorite?: boolean; +}; + +export type CustomizeSidebarSection = { + id: string; + title: string; + /** Items in DEFAULT order (favorites: saved order — that is their default). */ + items: CustomizeSidebarItem[]; +}; + +type SavedPreferences = { + sectionOrder?: string[]; + hiddenItems?: Record; + sectionItemOrder?: Record; +}; + +/** What Confirm produces; null clears a stored preference back to its default. */ +export type SidebarCustomizationPayload = { + sectionOrder: string[] | null; + hiddenItems: Record | null; + sectionItemOrder: Record | null; + favorites?: Array<{ id: string; label: string }>; + removedFavoriteIds?: string[]; +}; + +type DialogState = { + sectionOrder: string[]; + /** section id -> item ids in order */ + itemOrders: Record; + /** item id -> effective hidden */ + hidden: Record; + /** favorite id -> label being edited */ + labels: Record; + /** favorite ids staged for removal; applied on Confirm */ + removed: string[]; +}; + +const FAVORITES_SECTION_ID = "favorites"; +const ROW_HEIGHT = 44; + +function buildState( + sections: CustomizeSidebarSection[], + prefs: SavedPreferences | undefined +): DialogState { + const orderedSections = prefs ? orderByPreference(sections, prefs.sectionOrder) : sections; + + const itemOrders: Record = {}; + const hidden: Record = {}; + const labels: Record = {}; + + for (const section of sections) { + // Favorites' array order is canonical (already applied), so saved item order only applies to + // the static sections. + const orderedItems = + prefs && section.id !== FAVORITES_SECTION_ID + ? orderByPreference(section.items, prefs.sectionItemOrder?.[section.id]) + : section.items; + itemOrders[section.id] = orderedItems.map((item) => item.id); + + for (const item of section.items) { + hidden[item.id] = prefs + ? isItemHidden(item, prefs.hiddenItems) + : (item.defaultHidden ?? false); + if (item.isFavorite) { + labels[item.id] = item.name; + } + } + } + + return { + sectionOrder: orderedSections.map((section) => section.id), + itemOrders, + hidden, + labels, + removed: [], + }; +} + +function arraysEqual(a: string[], b: string[]) { + return a.length === b.length && a.every((value, index) => value === b[index]); +} + +/** + * The "Customize sidebar" modal: reorder sections (arrows), reorder items (drag), hide/show items + * (eye), and rename favorites inline. Nothing is applied until Confirm; Reset restores the default + * layout without touching which pages are favorited. + */ +export function CustomizeSidebarDialog({ + sections, + prefs, + onConfirm, + isConfirming, + confirmError, +}: { + sections: CustomizeSidebarSection[]; + prefs: SavedPreferences | undefined; + /** + * Owned by the parent: closing this dialog unmounts it, so it can't run its own fetcher. The + * parent submits the payload and closes the dialog once the save lands (or reports back via + * `confirmError`), so a failed save never silently reads as a successful one. + */ + onConfirm: (payload: SidebarCustomizationPayload) => void; + /** True from Confirm until the save (and the refreshed side menu data) lands. */ + isConfirming: boolean; + /** Save failure to surface next to Confirm; the dialog stays open for a retry. */ + confirmError?: string; +}) { + const [state, setState] = useState(() => buildState(sections, prefs)); + + // The Favorites section disappears with its last staged-removed favorite, matching the side + // menu (which hides the section when empty) + const displayedSections = (current: DialogState) => + current.sectionOrder + .map((id) => sections.find((section) => section.id === id)) + .filter((section): section is CustomizeSidebarSection => section !== undefined) + .filter( + (section) => + section.id !== FAVORITES_SECTION_ID || + section.items.some((item) => !current.removed.includes(item.id)) + ); + + const orderedSections = displayedSections(state); + + const moveSection = (sectionId: string, direction: -1 | 1) => { + setState((current) => { + // Swap with the DISPLAYED neighbor: a hidden Favorites entry may still sit in + // sectionOrder between two visible sections + const displayed = displayedSections(current).map((section) => section.id); + const neighborId = displayed[displayed.indexOf(sectionId) + direction]; + if (!neighborId) return current; + const next = [...current.sectionOrder]; + const a = next.indexOf(sectionId); + const b = next.indexOf(neighborId); + [next[a], next[b]] = [next[b], next[a]]; + return { ...current, sectionOrder: next }; + }); + }; + + const reorderItems = (sectionId: string, itemIds: string[]) => { + setState((current) => ({ + ...current, + itemOrders: { ...current.itemOrders, [sectionId]: itemIds }, + })); + }; + + const toggleHidden = (itemId: string) => { + setState((current) => ({ + ...current, + hidden: { ...current.hidden, [itemId]: !current.hidden[itemId] }, + })); + }; + + const setLabel = (itemId: string, label: string) => { + setState((current) => ({ ...current, labels: { ...current.labels, [itemId]: label } })); + }; + + const removeFavorite = (itemId: string) => { + setState((current) => ({ ...current, removed: [...current.removed, itemId] })); + }; + + // Reset restores the default layout (positions + visibility) but never touches favorite names + // or staged removals; Cancel is the way out of those + const reset = () => + setState((current) => ({ + ...buildState(sections, undefined), + labels: current.labels, + removed: current.removed, + })); + + const hasBlankLabels = sections.some((section) => + section.items.some( + (item) => + item.isFavorite && + !state.removed.includes(item.id) && + (state.labels[item.id] ?? item.name).trim().length === 0 + ) + ); + + const confirm = () => { + const defaults = buildState(sections, undefined); + + const hiddenOverrides: Record = {}; + for (const section of sections) { + for (const item of section.items) { + if (state.removed.includes(item.id)) continue; + const isHidden = state.hidden[item.id] ?? false; + if (isHidden !== (item.defaultHidden ?? false)) { + hiddenOverrides[item.id] = isHidden; + } + } + } + + const sectionItemOrder: Record = {}; + for (const section of sections) { + if (section.id === FAVORITES_SECTION_ID) continue; + const order = state.itemOrders[section.id] ?? []; + if (!arraysEqual(order, defaults.itemOrders[section.id] ?? [])) { + sectionItemOrder[section.id] = order; + } + } + + const favoritesSection = sections.find((section) => section.id === FAVORITES_SECTION_ID); + const favoriteOrder = (state.itemOrders[FAVORITES_SECTION_ID] ?? []).filter( + (id) => !state.removed.includes(id) + ); + const favoritesChanged = + favoritesSection !== undefined && + (state.removed.length > 0 || + !arraysEqual(favoriteOrder, defaults.itemOrders[FAVORITES_SECTION_ID] ?? []) || + favoritesSection.items.some( + (item) => + !state.removed.includes(item.id) && + (state.labels[item.id] ?? item.name).trim() !== item.name + )); + + // Parts equal to the defaults are sent as null so the stored preference is cleared, not pinned + const payload: SidebarCustomizationPayload = { + sectionOrder: arraysEqual(state.sectionOrder, defaults.sectionOrder) + ? null + : state.sectionOrder, + hiddenItems: Object.keys(hiddenOverrides).length > 0 ? hiddenOverrides : null, + sectionItemOrder: Object.keys(sectionItemOrder).length > 0 ? sectionItemOrder : null, + favorites: favoritesChanged + ? favoriteOrder.map((id) => ({ id, label: state.labels[id] ?? "" })) + : undefined, + removedFavoriteIds: state.removed.length > 0 ? state.removed : undefined, + }; + + onConfirm(payload); + }; + + return ( + + Customize sidebar + {/* Bleeds through the container's right padding (-mr-4/pr-4) so the scrollbar sits at the + modal edge, and through the vertical grid gaps (-mt-1.25/-mb-4) so the scrollport (and + scrollbar) starts at the header divider and ends at the footer border. pt-3/pb-3 are + INSIDE the scrollport: resting gaps around the list that content scrolls through. */} +
+ {orderedSections.map((section, index) => ( +
+
+ {section.title} +
+ moveSection(section.id, -1)} + > + + + moveSection(section.id, 1)} + > + + +
+
+ item.id)).filter( + (id) => !state.removed.includes(id) + )} + hidden={state.hidden} + labels={state.labels} + onReorder={(itemIds) => reorderItems(section.id, itemIds)} + onToggleHidden={toggleHidden} + onLabelChange={setLabel} + onRemove={removeFavorite} + /> +
+ ))} +
+ {/* Negative margins stretch the top divider across the modal's full width */} + +
+ + + + +
+
+ {confirmError && !isConfirming && ( + {confirmError} + )} + +
+ + + ); +} + +function SectionMoveButton({ + label, + disabled, + onClick, + children, +}: { + label: string; + disabled: boolean; + onClick: () => void; + children: React.ReactNode; +}) { + return ( + + ); +} + +function SectionItemList({ + section, + order, + hidden, + labels, + onReorder, + onToggleHidden, + onLabelChange, + onRemove, +}: { + section: CustomizeSidebarSection; + order: string[]; + hidden: Record; + labels: Record; + onReorder: (itemIds: string[]) => void; + onToggleHidden: (itemId: string) => void; + onLabelChange: (itemId: string, label: string) => void; + onRemove: (itemId: string) => void; +}) { + const { width, containerRef } = useContainerWidth({ initialWidth: 416 }); + + const items = order + .map((id) => section.items.find((item) => item.id === id)) + .filter((item): item is CustomizeSidebarItem => item !== undefined); + + const layout = items.map((item, index) => ({ i: item.id, x: 0, y: index, w: 1, h: 1 })); + + const handleDragStop = (nextLayout: Layout) => { + const sorted = [...nextLayout].sort((a, b) => a.y - b.y).map((entry) => entry.i); + if (!arraysEqual(sorted, order)) { + onReorder(sorted); + } + }; + + const renderRow = (item: CustomizeSidebarItem, options: { draggable: boolean }) => ( + onToggleHidden(item.id)} + onLabelChange={(label) => onLabelChange(item.id, label)} + onRemove={() => onRemove(item.id)} + /> + ); + + return ( +
}> + {items.length >= 2 ? ( + + {items.map((item) => ( +
{renderRow(item, { draggable: true })}
+ ))} + + ) : ( + items.map((item) =>
{renderRow(item, { draggable: false })}
) + )} +
+ ); +} + +function ModalItemRow({ + item, + isHidden, + label, + draggable, + onToggleHidden, + onLabelChange, + onRemove, +}: { + item: CustomizeSidebarItem; + isHidden: boolean; + label: string | undefined; + draggable: boolean; + onToggleHidden: () => void; + onLabelChange: (label: string) => void; + onRemove: () => void; +}) { + return ( +
+
+ + {item.isFavorite ? ( + <> + onLabelChange(e.target.value)} + variant="medium" + maxLength={64} + containerClassName="max-w-60" + aria-label={`Rename ${item.name}`} + /> + {(label ?? item.name).trim().length === 0 && ( + Name can't be blank + )} + + ) : ( + {item.name} + )} +
+
+ {item.isFavorite && ( + + )} + + {draggable ? ( +
+ +
+ ) : ( +
+ )} +
+
+ ); +} diff --git a/apps/webapp/app/components/navigation/DashboardList.tsx b/apps/webapp/app/components/navigation/DashboardList.tsx index 7f3b6e3bacb..39a7de00c1d 100644 --- a/apps/webapp/app/components/navigation/DashboardList.tsx +++ b/apps/webapp/app/components/navigation/DashboardList.tsx @@ -184,6 +184,7 @@ function DashboardChildMenuItem({ to={item.path} isCollapsed={isCollapsed} disableIconHover + yieldActiveToFavorite action={ showDragHandle ? (
diff --git a/apps/webapp/app/components/navigation/FavoritePageButton.tsx b/apps/webapp/app/components/navigation/FavoritePageButton.tsx new file mode 100644 index 00000000000..c0d03453b38 --- /dev/null +++ b/apps/webapp/app/components/navigation/FavoritePageButton.tsx @@ -0,0 +1,144 @@ +import { StarIcon as StarIconOutline } from "@heroicons/react/24/outline"; +import { StarIcon as StarIconSolid } from "@heroicons/react/20/solid"; +import { useFetcher, useLocation, useSearchParams } from "@remix-run/react"; +import { useEffect } from "react"; +import { useIsImpersonating } from "~/hooks/useOrganizations"; +import { useShortcutKeys } from "~/hooks/useShortcutKeys"; +import { useOptionalUser } from "~/hooks/useUser"; +import { cn } from "~/utils/cn"; +import { Button } from "../primitives/Buttons"; +import { ShortcutKey } from "../primitives/ShortcutKey"; +import { SimpleTooltip } from "../primitives/Tooltip"; +import { + buildFavoriteLabel, + canonicalFavoriteUrl, + FAVORITE_SEARCH_PARAM, + FAVORITES_ACTION_PATH, + favoritePageUrl, + resolvePageMeta, + useFavorites, +} from "./favoritePages"; + +/** + * The star in the page header that favorites the current page (full URL, including filters and + * tabs) to the side menu. Toggled by click or Option+F. + */ +export function FavoritePageButton({ + pageTitle, + className, +}: { + pageTitle?: string; + className?: string; +}) { + const user = useOptionalUser(); + const isImpersonating = useIsImpersonating(); + const location = useLocation(); + const favorites = useFavorites(); + const fetcher = useFetcher(); + const [, setSearchParams] = useSearchParams(); + + // The marker param and pagination position never count toward URL identity, so paging through + // a favorited view keeps the same favorite (and never saves a soon-stale cursor) + const url = favoritePageUrl(location.pathname, location.search); + + // A marker that isn't one of this user's favorites came from a shared link (or a favorite + // that's since been removed): clean it from the URL so the page behaves like a normal visit. + const marker = new URLSearchParams(location.search).get(FAVORITE_SEARCH_PARAM); + const hasForeignMarker = + user !== undefined && marker !== null && !favorites.some((f) => f.id === marker); + + useEffect(() => { + if (!hasForeignMarker) return; + setSearchParams( + (previous) => { + const next = new URLSearchParams(previous); + next.delete(FAVORITE_SEARCH_PARAM); + return next; + }, + { replace: true, preventScrollReset: true } + ); + }, [hasForeignMarker, setSearchParams]); + const existing = favorites.find((favorite) => canonicalFavoriteUrl(favorite.url) === url); + const isFavorited = existing !== undefined; + // The tooltip names the favorite: its custom name once saved, else the label saving would use + // (which includes detail-page ids and filter summaries, e.g. "Runs: Completed, last 7d") + const pageName = + existing?.label ?? buildFavoriteLabel(location.pathname, location.search, pageTitle); + + const toggle = () => { + if (existing) { + fetcher.submit( + { intent: "remove", id: existing.id }, + { method: "POST", action: FAVORITES_ACTION_PATH } + ); + } else { + fetcher.submit( + { + intent: "add", + id: crypto.randomUUID(), + url, + label: buildFavoriteLabel(location.pathname, location.search, pageTitle), + icon: resolvePageMeta(location.pathname).icon, + }, + { method: "POST", action: FAVORITES_ACTION_PATH } + ); + } + }; + + const showButton = user !== undefined && !isImpersonating; + + // Option+F reports event.key "ƒ" on macOS, but the hotkeys matcher falls back to the physical + // event.code ("KeyF"), so the standard hook captures it; exact modifier matching keeps the + // bare "f" filter shortcut separate. + useShortcutKeys({ + shortcut: { key: "f", modifiers: ["alt"] }, + action: (event) => { + event.preventDefault(); + toggle(); + }, + disabled: !showButton, + }); + + if (!showButton) { + return null; + } + + const tooltipLabel = isFavorited + ? `Remove ${pageName} from favorites` + : `Add ${pageName} to favorites`; + + return ( + +
@@ -1097,10 +1270,274 @@ export function SideMenu({
+ { + setCustomizeOpen(open); + if (!open) { + // Cancel/ESC during a pending confirm abandons the wait; a still-running save is + // harmless (the menu revalidates whenever it lands) + setCustomizeConfirmPending(false); + setCustomizeError(undefined); + } + }} + > + {/* Mounted only while open so the modal state re-seeds from current preferences each time */} + {isCustomizeOpen && ( + + )} +
); } +/** A section built from {@link SideMenuItemConfig}s with the user's order/hidden prefs applied. */ +function CustomizableSideMenuSection({ + section, + itemOrder, + hiddenItems, + isCollapsed, + initialCollapsed, + onCollapseToggle, + headerMenu, + onCustomize, +}: { + section: SideMenuSectionConfig; + itemOrder: string[] | undefined; + hiddenItems: Record | undefined; + isCollapsed: boolean; + initialCollapsed: boolean; + onCollapseToggle: (collapsed: boolean) => void; + headerMenu: ReactNode; + /** Undefined when customizing isn't offered (impersonated sessions can't persist a layout). */ + onCustomize: (() => void) | undefined; +}) { + const orderedItems = orderByPreference(section.items, itemOrder); + const visibleItems = orderedItems.filter((item) => !isItemHidden(item, hiddenItems)); + const moreItems = orderedItems.filter((item) => isItemHidden(item, hiddenItems)); + + return ( + + {visibleItems.map((item) => ( + + + {item.after} + + ))} + {moreItems.length > 0 && ( + ({ + id: item.id, + name: item.name, + icon: item.icon, + to: item.to, + }))} + isCollapsed={isCollapsed} + onCustomize={onCustomize} + /> + )} + + ); +} + +/** The user's favorited pages; hidden favorites collapse into the trailing "More" item. */ +function FavoritesSideMenuSection({ + favorites, + hiddenItems, + isCollapsed, + initialCollapsed, + onCollapseToggle, + headerMenu, + onCustomize, + onRemoveFavorite, + onRenameFavorite, +}: { + favorites: FavoritePage[]; + hiddenItems: Record | undefined; + isCollapsed: boolean; + initialCollapsed: boolean; + onCollapseToggle: (collapsed: boolean) => void; + headerMenu: ReactNode; + /** Undefined when customizing isn't offered (impersonated sessions can't persist a layout). */ + onCustomize: (() => void) | undefined; + onRemoveFavorite: (id: string) => void; + onRenameFavorite: (id: string, label: string) => void; +}) { + const visible = favorites.filter((favorite) => !(hiddenItems?.[favorite.id] ?? false)); + const hidden = favorites.filter((favorite) => hiddenItems?.[favorite.id] ?? false); + + return ( + + {visible.map((favorite) => ( + + ))} + {hidden.length > 0 && ( + ({ + id: favorite.id, + name: favorite.label, + icon: favoritePageIcon(favorite.icon), + iconClassName: favoritePageIconClassName(favorite.icon), + to: favoriteLinkTo(favorite), + }))} + isCollapsed={isCollapsed} + onCustomize={onCustomize} + /> + )} + + ); +} + +/** + * The trailing "More" item of a section: a popover listing that section's hidden items. Only + * rendered when the section has hidden items. + */ +function SideMenuMoreItem({ + items, + isCollapsed, + onCustomize, +}: { + items: Array<{ id: string; name: string; icon: RenderIcon; iconClassName?: string; to: string }>; + isCollapsed: boolean; + /** Undefined when customizing isn't offered (impersonated sessions can't persist a layout). */ + onCustomize: (() => void) | undefined; +}) { + const [isOpen, setOpen] = useState(false); + const navigation = useNavigation(); + + // Watch search too: navigating to a favorite can change only the search on the same pathname + useEffect(() => { + setOpen(false); + }, [navigation.location?.pathname, navigation.location?.search]); + + return ( + + + + + More + + + } + content="More" + side="right" + sideOffset={8} + hidden={!isCollapsed} + asChild + tabbable + disableHoverableContent + /> + +
+ {items.map((item) => ( + + ))} +
+ {/* flex-col blockifies the inline-block menu item, avoiding stray line-box space below */} + {onCustomize && ( +
+ { + setOpen(false); + onCustomize(); + }} + /> +
+ )} + + + ); +} + +/** The ellipsis on section headers (visible on hover) opening the sidebar customization menu. */ +function SectionHeaderMenu({ onCustomize }: { onCustomize: () => void }) { + const [isOpen, setOpen] = useState(false); + + return ( + + + + + + {/* flex-col blockifies the inline-block menu item, avoiding stray line-box space below */} +
+ { + setOpen(false); + onCustomize(); + }} + /> +
+ + + ); +} + function V3DeprecationPanel({ isCollapsed, isV3, diff --git a/apps/webapp/app/components/navigation/SideMenuItem.tsx b/apps/webapp/app/components/navigation/SideMenuItem.tsx index f17051266d4..035fcc54370 100644 --- a/apps/webapp/app/components/navigation/SideMenuItem.tsx +++ b/apps/webapp/app/components/navigation/SideMenuItem.tsx @@ -3,6 +3,9 @@ import { type ButtonHTMLAttributes, forwardRef, type ReactNode, + useEffect, + useRef, + useState, } from "react"; import { Link } from "@remix-run/react"; import { motion } from "framer-motion"; @@ -10,6 +13,60 @@ import { usePathName } from "~/hooks/usePathName"; import { cn } from "~/utils/cn"; import { type RenderIcon, Icon } from "../primitives/Icon"; import { SimpleTooltip } from "../primitives/Tooltip"; +import { useActiveFavoriteId } from "./favoritePages"; + +/** Right-edge fade shown instead of a hard clip, only while the label actually overflows. */ +const LABEL_OVERFLOW_MASK = "linear-gradient(to right, black calc(100% - 1.5rem), transparent)"; + +/** + * A menu label that fades out at its right edge when (and only when) the text overflows. Text + * that fits renders exactly as before, with no mask. Overflow is re-measured when the element + * resizes (e.g. while drag-resizing the menu) and when the label text changes. + */ +export function SideMenuLabel({ + children, + className, + style, +}: { + children: ReactNode; + className?: string; + style?: React.CSSProperties; +}) { + const ref = useRef(null); + const [isOverflowing, setIsOverflowing] = useState(false); + const checkRef = useRef<() => void>(() => {}); + + useEffect(() => { + const el = ref.current; + if (!el) return; + const check = () => setIsOverflowing(el.scrollWidth > el.clientWidth + 1); + checkRef.current = check; + check(); + const observer = new ResizeObserver(check); + observer.observe(el); + return () => observer.disconnect(); + }, []); + + // Re-measure when the text changes (a rename can flip overflow without resizing the element) + useEffect(() => { + checkRef.current(); + }, [children]); + + return ( + + {children} + + ); +} export function SideMenuItem({ icon, @@ -27,6 +84,8 @@ export function SideMenuItem({ action, disableIconHover = false, indented = false, + isActive: isActiveOverride, + yieldActiveToFavorite = false, "data-action": dataAction, }: { icon?: RenderIcon; @@ -45,10 +104,24 @@ export function SideMenuItem({ disableIconHover?: boolean; /** Indented variant for grouped sub-items; only applied when the menu is expanded. */ indented?: boolean; + /** Overrides the default pathname === to active check (e.g. favorites match on full URL). */ + isActive?: boolean; + /** + * In menus that render the Favorites section (the main project menu), an active favorite owns + * the highlight, so the plain item yields its active state to it. Menus without a favorites + * list (org settings, account) must not set this: they have no favorite item to carry the + * highlight instead. + */ + yieldActiveToFavorite?: boolean; "data-action"?: string; }) { const pathName = usePathName(); - const isActive = pathName === to; + // Only the user's OWN favorites own a view (via the marker param); markers from shared links + // don't count (see useActiveFavoriteId). + const activeFavoriteId = useActiveFavoriteId(); + const isActive = + isActiveOverride ?? + (pathName === to && (!yieldActiveToFavorite || activeFavoriteId === undefined)); const isIndented = indented && !isCollapsed; @@ -92,14 +165,14 @@ export function SideMenuItem({ className="flex w-full min-w-0 items-center justify-between" style={{ opacity: "var(--sm-label-opacity, 1)" }} > - {name} - + {badge && !isCollapsed && (
{badge}
)} @@ -189,9 +262,9 @@ export const SideMenuItemButton = forwardRef< icon={icon} className="size-5 shrink-0 text-text-dimmed group-hover/menuitem:text-text-bright" /> - + {name} - + {trailing && {trailing}} ); diff --git a/apps/webapp/app/components/navigation/SideMenuSection.tsx b/apps/webapp/app/components/navigation/SideMenuSection.tsx index 8225d9e701c..4f0d37e2716 100644 --- a/apps/webapp/app/components/navigation/SideMenuSection.tsx +++ b/apps/webapp/app/components/navigation/SideMenuSection.tsx @@ -12,6 +12,11 @@ type Props = { itemSpacingClassName?: string; /** Optional action element (e.g., + button) to render on the right side of the header */ headerAction?: React.ReactNode; + /** + * Optional menu (e.g. an ellipsis popover) overlaid on the right of the header. Only visible + * while hovering the header row, or while its popover is open. + */ + headerMenu?: React.ReactNode; }; /** A collapsible section for the side menu. Collapsed state is controlled via props + a toggle callback. */ @@ -23,6 +28,7 @@ export function SideMenuSection({ isSideMenuCollapsed = false, itemSpacingClassName = "space-y-px", headerAction, + headerMenu, }: Props) { const [isCollapsed, setIsCollapsed] = useState(initialCollapsed); const contentRef = useRef(null); @@ -45,7 +51,7 @@ export function SideMenuSection({ return (
{/* Header container - stays in DOM to preserve height */} -
+
{/* Header fades out as the menu narrows via --sm-label-opacity (falls back to 1 unset). Hover background and text color snap (no transition), matching the nav items. @@ -53,8 +59,9 @@ export function SideMenuSection({ + {headerMenu !== undefined && + !isSideMenuCollapsed && ( + // Outer div fades with the labels (inline style would defeat the hover opacity classes + // on the inner div, so they're split). +
+ {/* focus-within keeps the trigger visible for keyboard users tabbing onto it */} +
+ {headerMenu} +
+
+ )} {/* Divider fades in via --sm-collapse (0 → 1) as the header fades out. Only while expanded. */} diff --git a/apps/webapp/app/components/navigation/favoritePages.tsx b/apps/webapp/app/components/navigation/favoritePages.tsx new file mode 100644 index 00000000000..4439aadedfd --- /dev/null +++ b/apps/webapp/app/components/navigation/favoritePages.tsx @@ -0,0 +1,511 @@ +import { BeakerIcon } from "@heroicons/react/24/outline"; +import { IconChartHistogram } from "@tabler/icons-react"; +import { useFetchers, useLocation } from "@remix-run/react"; +import { ClockIcon } from "~/assets/icons/ClockIcon"; +import { CubeSparkleIcon } from "~/assets/icons/CubeSparkleIcon"; +import { TaskIconSmall } from "~/assets/icons/TaskIcon"; +import { AIChatIcon } from "~/assets/icons/AIChatIcon"; +import { AIMetricsIcon } from "~/assets/icons/AIMetricsIcon"; +import { AIPenIcon } from "~/assets/icons/AIPenIcon"; +import { AvatarCircleIcon } from "~/assets/icons/AvatarCircleIcon"; +import { BatchesIcon } from "~/assets/icons/BatchesIcon"; +import { BellIcon } from "~/assets/icons/BellIcon"; +import { Box3DIcon } from "~/assets/icons/Box3DIcon"; +import { BugIcon } from "~/assets/icons/BugIcon"; +import { ChainLinkIcon } from "~/assets/icons/ChainLinkIcon"; +import { ChartArrowIcon } from "~/assets/icons/ChartArrowIcon"; +import { ChartBarIcon } from "~/assets/icons/ChartBarIcon"; +import { CodeSquareIcon } from "~/assets/icons/CodeSquareIcon"; +import { ConcurrencyIcon } from "~/assets/icons/ConcurrencyIcon"; +import { CreditCardIcon } from "~/assets/icons/CreditCardIcon"; +import { DeploymentsIcon } from "~/assets/icons/DeploymentsIcon"; +import { DialIcon } from "~/assets/icons/DialIcon"; +import { BranchEnvironmentIconSmall } from "~/assets/icons/EnvironmentIcons"; +import { FolderOpenIcon } from "~/assets/icons/FolderOpenIcon"; +import { GlobeLinesIcon } from "~/assets/icons/GlobeLinesIcon"; +import { IDIcon } from "~/assets/icons/IDIcon"; +import { IntegrationsIcon } from "~/assets/icons/IntegrationsIcon"; +import { KeyIcon } from "~/assets/icons/KeyIcon"; +import { ListCheckedIcon } from "~/assets/icons/ListCheckedIcon"; +import { LogsIcon } from "~/assets/icons/LogsIcon"; +import { PadlockIcon } from "~/assets/icons/PadlockIcon"; +import { QueuesIcon } from "~/assets/icons/QueuesIcon"; +import { RolesIcon } from "~/assets/icons/RolesIcon"; +import { RunsIcon } from "~/assets/icons/RunsIcon"; +import { ShieldIcon } from "~/assets/icons/ShieldIcon"; +import { SlackIcon } from "~/assets/icons/SlackIcon"; +import { SlidersIcon } from "~/assets/icons/SlidersIcon"; +import { StarIcon } from "~/assets/icons/StarIcon"; +import { TasksIcon } from "~/assets/icons/TasksIcon"; +import { UsageIcon } from "~/assets/icons/UsageIcon"; +import { UserGroupIcon } from "~/assets/icons/UserGroupIcon"; +import { WaitpointTokenIcon } from "~/assets/icons/WaitpointTokenIcon"; +import { VercelLogo } from "~/components/integrations/VercelLogo"; +import { useOptionalUser } from "~/hooks/useUser"; +import { type FavoritePage } from "~/services/dashboardPreferences.server"; +import { type RenderIcon } from "../primitives/Icon"; + +export const FAVORITES_ACTION_PATH = "/resources/preferences/favorites"; + +/** + * Marker search param appended to favorite links. It makes a favorited URL distinct from its + * plain counterpart, so only the favorite (never the matching main menu item) highlights as + * active, and the marker identifies WHICH favorite when several share a pathname. + */ +export const FAVORITE_SEARCH_PARAM = "fav"; + +/** + * Icons a favorited page can be saved with, keyed by a stable string so preferences never store + * component references, plus the icon color used when the favorite is the active page and an + * optional size override for icons drawn without internal padding (brand logos). Unknown keys + * fall back to the star. + */ +const FAVORITE_PAGE_ICONS: Record< + string, + { icon: RenderIcon; activeColor: string; className?: string } +> = { + tasks: { icon: TasksIcon, activeColor: "text-tasks" }, + // Task detail pages carry their task-type icon, matching TaskTriggerSourceIcon + "task-standard": { icon: TaskIconSmall, activeColor: "text-tasks" }, + "task-scheduled": { icon: ClockIcon, activeColor: "text-schedules" }, + "task-agent": { icon: CubeSparkleIcon, activeColor: "text-agents" }, + runs: { icon: RunsIcon, activeColor: "text-runs" }, + sessions: { icon: AIChatIcon, activeColor: "text-sessions" }, + prompts: { icon: AIPenIcon, activeColor: "text-aiPrompts" }, + models: { icon: Box3DIcon, activeColor: "text-models" }, + logs: { icon: LogsIcon, activeColor: "text-logs" }, + errors: { icon: BugIcon, activeColor: "text-errors" }, + query: { icon: CodeSquareIcon, activeColor: "text-query" }, + queues: { icon: QueuesIcon, activeColor: "text-queues" }, + dashboards: { icon: ChartBarIcon, activeColor: "text-metrics" }, + "run-metrics": { icon: ChartArrowIcon, activeColor: "text-runs" }, + "ai-metrics": { icon: AIMetricsIcon, activeColor: "text-aiMetrics" }, + "custom-dashboard": { icon: IconChartHistogram, activeColor: "text-text-bright" }, + deployments: { icon: DeploymentsIcon, activeColor: "text-deployments" }, + "environment-variables": { icon: IDIcon, activeColor: "text-environmentVariables" }, + branches: { icon: BranchEnvironmentIconSmall, activeColor: "text-previewBranches" }, + regions: { icon: GlobeLinesIcon, activeColor: "text-regions" }, + waitpoints: { icon: WaitpointTokenIcon, activeColor: "text-sky-500" }, + batches: { icon: BatchesIcon, activeColor: "text-batches" }, + "bulk-actions": { icon: ListCheckedIcon, activeColor: "text-text-bright" }, + apikeys: { icon: KeyIcon, activeColor: "text-text-bright" }, + alerts: { icon: BellIcon, activeColor: "text-text-bright" }, + concurrency: { icon: ConcurrencyIcon, activeColor: "text-text-bright" }, + limits: { icon: DialIcon, activeColor: "text-text-bright" }, + schedules: { icon: ClockIcon, activeColor: "text-schedules" }, + test: { icon: BeakerIcon, activeColor: "text-text-bright" }, + "project-settings": { icon: SlidersIcon, activeColor: "text-text-bright" }, + integrations: { icon: IntegrationsIcon, activeColor: "text-text-bright" }, + // Brand logos have no internal padding, so they render one step smaller (matching the org menu) + slack: { icon: SlackIcon, activeColor: "text-text-bright", className: "size-4" }, + vercel: { icon: VercelLogo, activeColor: "text-text-bright", className: "size-4" }, + project: { icon: FolderOpenIcon, activeColor: "text-text-bright" }, + "org-settings": { icon: SlidersIcon, activeColor: "text-text-bright" }, + team: { icon: UserGroupIcon, activeColor: "text-text-bright" }, + billing: { icon: CreditCardIcon, activeColor: "text-text-bright" }, + usage: { icon: UsageIcon, activeColor: "text-text-bright" }, + roles: { icon: RolesIcon, activeColor: "text-text-bright" }, + sso: { icon: PadlockIcon, activeColor: "text-text-bright" }, + "private-connections": { icon: ChainLinkIcon, activeColor: "text-text-bright" }, + account: { icon: AvatarCircleIcon, activeColor: "text-text-bright" }, + tokens: { icon: ShieldIcon, activeColor: "text-text-bright" }, + security: { icon: PadlockIcon, activeColor: "text-text-bright" }, + page: { icon: StarIcon, activeColor: "text-text-bright" }, +}; + +export function favoritePageIcon(iconKey: string | undefined): RenderIcon { + return (iconKey ? FAVORITE_PAGE_ICONS[iconKey]?.icon : undefined) ?? StarIcon; +} + +export function favoritePageActiveColor(iconKey: string | undefined): string { + return (iconKey ? FAVORITE_PAGE_ICONS[iconKey]?.activeColor : undefined) ?? "text-text-bright"; +} + +/** Size override for favorite icons that need one (see FAVORITE_PAGE_ICONS). */ +export function favoritePageIconClassName(iconKey: string | undefined): string | undefined { + return iconKey ? FAVORITE_PAGE_ICONS[iconKey]?.className : undefined; +} + +/** Href for a favorite: its saved URL plus the marker param (see FAVORITE_SEARCH_PARAM). */ +export function favoriteLinkTo(favorite: FavoritePage): string { + const [path, search = ""] = favorite.url.split("?"); + const params = new URLSearchParams(search); + params.set(FAVORITE_SEARCH_PARAM, favorite.id); + return `${path}?${params.toString()}`; +} + +/** Pagination position params: never part of a favorite's identity (see favoritePageUrl). */ +const PAGINATION_PARAMS = ["cursor", "direction", "page"]; + +/** + * The canonical URL a favorite saves and matches against: the path and search minus the favorite + * marker (presentation-only) and the pagination position (cursors go stale, and page N of a view + * is not a different view). A favorite pins filters and tabs, never a transient page of them. + */ +export function favoritePageUrl(pathname: string, search: string): string { + const params = new URLSearchParams(search); + params.delete(FAVORITE_SEARCH_PARAM); + for (const param of PAGINATION_PARAMS) { + params.delete(param); + } + const result = params.toString(); + return pathname + (result.length > 0 ? `?${result}` : ""); +} + +/** favoritePageUrl for an already-joined URL, e.g. a favorite's stored one (which may predate + * pagination stripping). */ +export function canonicalFavoriteUrl(url: string): string { + const [pathname, search = ""] = url.split("?"); + return favoritePageUrl(pathname, search); +} + +/** + * A favorite is active only while the URL is the view it saved: its marker param is present AND + * the canonical URL still matches. Changing any filter on the page diverges the URL from the + * favorite, so it deactivates (and the regular menu item takes over) — but paging within the + * view keeps it active, matching what the favorite pins. + */ +export function isFavoriteActive( + favorite: FavoritePage, + pathname: string, + search: string +): boolean { + return ( + new URLSearchParams(search).get(FAVORITE_SEARCH_PARAM) === favorite.id && + canonicalFavoriteUrl(favorite.url) === favoritePageUrl(pathname, search) + ); +} + +/** + * The id of the favorite driving the current view: the URL's marker param, but only when it + * belongs to one of the current user's favorites AND the URL still matches that favorite's + * saved view. A marker from someone else's shared link, a removed favorite's stale link, or a + * view whose filters have since been changed resolves to undefined, so regular menu + * highlighting applies. + */ +export function useActiveFavoriteId(): string | undefined { + const location = useLocation(); + const favorites = useFavorites(); + + const marker = new URLSearchParams(location.search).get(FAVORITE_SEARCH_PARAM); + if (!marker) return undefined; + const favorite = favorites.find((f) => f.id === marker); + if (!favorite) return undefined; + return isFavoriteActive(favorite, location.pathname, location.search) ? marker : undefined; +} + +type PageMeta = { + /** Key into FAVORITE_PAGE_ICONS. */ + icon: string; + /** The page's name as shown in navigation, e.g. "Queues". */ + name: string; + /** Singular label-prefix for detail pages, e.g. "Queue" -> "Queue: my-queue". */ + singular?: string; + /** + * Entity name taken from the URL, used verbatim as the label. For detail pages whose header + * title is composed JSX (task/agent pages render an icon + slug), so no plain-text title + * reaches the star. The icon already conveys the type, so no prefix is added. + */ + entityName?: string; +}; + +const ENV_PAGE_META: Record = { + "": { icon: "tasks", name: "Tasks", singular: "Task" }, + runs: { icon: "runs", name: "Runs", singular: "Run" }, + sessions: { icon: "sessions", name: "Sessions", singular: "Session" }, + prompts: { icon: "prompts", name: "Prompts", singular: "Prompt" }, + models: { icon: "models", name: "Models", singular: "Model" }, + logs: { icon: "logs", name: "Logs" }, + errors: { icon: "errors", name: "Errors", singular: "Error" }, + query: { icon: "query", name: "Query" }, + queues: { icon: "queues", name: "Queues", singular: "Queue" }, + dashboards: { icon: "dashboards", name: "Dashboards", singular: "Dashboard" }, + deployments: { icon: "deployments", name: "Deploys", singular: "Deploy" }, + "environment-variables": { icon: "environment-variables", name: "Environment variables" }, + branches: { icon: "branches", name: "Preview branches", singular: "Branch" }, + regions: { icon: "regions", name: "Regions" }, + waitpoints: { icon: "waitpoints", name: "Waitpoint tokens", singular: "Waitpoint" }, + batches: { icon: "batches", name: "Batches", singular: "Batch" }, + "bulk-actions": { icon: "bulk-actions", name: "Bulk actions", singular: "Bulk action" }, + apikeys: { icon: "apikeys", name: "API keys" }, + alerts: { icon: "alerts", name: "Alerts", singular: "Alert" }, + concurrency: { icon: "concurrency", name: "Concurrency" }, + limits: { icon: "limits", name: "Limits" }, + schedules: { icon: "schedules", name: "Schedules", singular: "Schedule" }, + test: { icon: "test", name: "Test", singular: "Test" }, + // The playground route is the Test page too (its header reads "Test") + playground: { icon: "test", name: "Test", singular: "Test" }, +}; + +const ORG_SETTINGS_PAGE_META: Record = { + "": { icon: "org-settings", name: "Organization settings" }, + team: { icon: "team", name: "Team" }, + billing: { icon: "billing", name: "Billing" }, + "billing-limits": { icon: "alerts", name: "Billing alerts" }, + usage: { icon: "usage", name: "Usage" }, + roles: { icon: "roles", name: "Roles" }, + sso: { icon: "sso", name: "SSO" }, + "private-connections": { icon: "private-connections", name: "Private connections" }, + integrations: { icon: "integrations", name: "Integrations" }, + danger: { icon: "org-settings", name: "Danger zone" }, +}; + +const ACCOUNT_PAGE_META: Record = { + "": { icon: "account", name: "Profile" }, + tokens: { icon: "tokens", name: "Personal Access Tokens" }, + security: { icon: "security", name: "Security" }, +}; + +/** Best-effort icon + name for any dashboard page, derived from its URL shape. */ +export function resolvePageMeta(pathname: string): PageMeta { + const envMatch = pathname.match(/^\/orgs\/[^/]+\/projects\/[^/]+\/env\/[^/]+(?:\/([^?]*))?$/); + if (envMatch) { + const segments = (envMatch[1] ?? "").split("/").filter(Boolean); + const first = segments[0] ?? ""; + if (first === "settings") { + return segments[1] === "integrations" + ? { icon: "integrations", name: "Integrations" } + : { icon: "project-settings", name: "Project settings" }; + } + if (first === "dashboards") { + // The built-in metric dashboards and custom dashboards have their own identities (and icons) + if (segments[1] === "overview") return { icon: "run-metrics", name: "Run metrics" }; + if (segments[1] === "llm") return { icon: "ai-metrics", name: "AI metrics" }; + if (segments[1] === "custom") { + return { icon: "custom-dashboard", name: "Dashboards", singular: "Dashboard" }; + } + } + + // Task detail: /tasks/{standard|scheduled}/{slug}. The slug is the only place the task name + // exists (the page header renders it as JSX), so it becomes the label. + if (first === "tasks" && segments[2]) { + const slug = decodeURIComponent(segments[2]); + return segments[1] === "scheduled" + ? { icon: "task-scheduled", name: "Scheduled task", entityName: slug } + : { icon: "task-standard", name: "Standard task", entityName: slug }; + } + + // Agent tasks live outside /tasks: /agents/{slug} + if (first === "agents") { + return segments[1] + ? { + icon: "task-agent", + name: "Agent task", + entityName: decodeURIComponent(segments[1]), + } + : { icon: "task-agent", name: "Agents" }; + } + + return ENV_PAGE_META[first] ?? { icon: "page", name: "Page" }; + } + + const orgSettingsMatch = pathname.match(/^\/orgs\/[^/]+\/settings(?:\/([^?]*))?$/); + if (orgSettingsMatch) { + const segments = (orgSettingsMatch[1] ?? "").split("/").filter(Boolean); + if (segments[0] === "integrations" && segments[1] === "slack") { + return { icon: "slack", name: "Slack integration" }; + } + if (segments[0] === "integrations" && segments[1] === "vercel") { + return { icon: "vercel", name: "Vercel integration" }; + } + return ORG_SETTINGS_PAGE_META[segments[0] ?? ""] ?? { icon: "org-settings", name: "Settings" }; + } + + if (/^\/orgs\/[^/]+\/projects\/[^/]+/.test(pathname)) { + return { icon: "project", name: "Project" }; + } + + if (/^\/orgs\/[^/]+/.test(pathname)) { + return { icon: "project", name: "Projects" }; + } + + const accountMatch = pathname.match(/^\/account(?:\/([^?]*))?$/); + if (accountMatch) { + const segments = (accountMatch[1] ?? "").split("/").filter(Boolean); + return ACCOUNT_PAGE_META[segments[0] ?? ""] ?? { icon: "account", name: "Account" }; + } + + return { icon: "page", name: "Page" }; +} + +const MAX_LABEL_LENGTH = 50; + +function truncateLabel(label: string): string { + return label.length > MAX_LABEL_LENGTH ? `${label.slice(0, MAX_LABEL_LENGTH - 1)}…` : label; +} + +/** + * Short id for a detail page whose last URL segment is a friendly id ("run_cmryyza…05hrqq9n"). + * Uses the same 8-character tail the dashboard tables display, so the label matches what the + * user sees elsewhere. + */ +function detailIdFromPath(pathname: string): string | undefined { + const segments = pathname.split("/").filter(Boolean); + const last = segments[segments.length - 1]; + if (last && /^(run|batch|session|deployment|schedule|waitpoint)_[a-z0-9]{8,}$/i.test(last)) { + return last.slice(-8); + } + return undefined; +} + +/** Task type filter on the Tasks page (?types=…) becomes the whole favorite name. */ +const TASK_TYPE_LABELS: Record = { + AGENT: "Agent tasks", + STANDARD: "Standard tasks", + SCHEDULED: "Scheduled tasks", +}; + +/** "COMPLETED_SUCCESSFULLY" -> "Completed successfully", "history" -> "History". */ +function humanizeValue(value: string): string { + const lowered = value.toLowerCase().replaceAll("_", " "); + return lowered.charAt(0).toUpperCase() + lowered.slice(1); +} + +/** Pagination/UI-state params that never describe what the user filtered. */ +const NON_FILTER_PARAMS = [FAVORITE_SEARCH_PARAM, ...PAGINATION_PARAMS, "span"]; + +/** + * Summarize a filtered view's search params into a short, selective descriptor for the favorite + * label ("Completed successfully, last 7d +2"). The best-known filters are named (at most two); + * everything else only counts toward a "+N" so heavily filtered views stay readable. + */ +function describeFilters(search: string): string | undefined { + const params = new URLSearchParams(search); + for (const param of NON_FILTER_PARAMS) { + params.delete(param); + } + + const parts: string[] = []; + const consumed = new Set(); + + const take = (key: string, describe: (values: string[]) => string | undefined) => { + const values = params.getAll(key).filter((value) => value.length > 0); + if (values.length === 0) return; + consumed.add(key); + const described = describe(values); + if (described) parts.push(described); + }; + + // Priority order: the filters most likely to identify the view come first + take("statuses", (v) => (v.length === 1 ? humanizeValue(v[0]) : `${v.length} statuses`)); + take("levels", (v) => (v.length === 1 ? humanizeValue(v[0]) : `${v.length} levels`)); + take("tasks", (v) => (v.length === 1 ? v[0] : `${v.length} tasks`)); + take("queues", (v) => (v.length === 1 ? v[0].replace(/^task\//, "") : `${v.length} queues`)); + take("tags", (v) => (v.length === 1 ? v[0] : `${v.length} tags`)); + take("period", (v) => `last ${v[0]}`); + if (params.has("from") || params.has("to")) { + consumed.add("from"); + consumed.add("to"); + parts.push("custom range"); + } + take("versions", (v) => (v.length === 1 ? v[0] : `${v.length} versions`)); + take("machines", (v) => (v.length === 1 ? v[0] : `${v.length} machines`)); + take("tab", (v) => humanizeValue(v[0])); + // The runs list appends rootOnly=false by default; only the non-default value is a filter + take("rootOnly", (v) => (v[0] === "true" ? "root only" : undefined)); + + const remaining = new Set([...params.keys()].filter((key) => !consumed.has(key))).size; + + const MAX_NAMED_PARTS = 2; + const shown = parts.slice(0, MAX_NAMED_PARTS); + const extra = parts.length - shown.length + remaining; + + if (shown.length === 0) { + return extra > 0 ? `${extra} filter${extra === 1 ? "" : "s"}` : undefined; + } + return shown.join(", ") + (extra > 0 ? ` +${extra}` : ""); +} + +/** + * Compose the default side menu label for a favorited page. Plain list pages keep their nav + * name ("Queues"); detail pages get an identifying prefix ("Queue: email-queue", or the short + * id for friendly-id pages: "Run: 05hrqq9n"); filtered views summarize their filters ("Runs: + * Completed successfully, last 7d"). Users can always rename. + */ +export function buildFavoriteLabel( + pathname: string, + search: string, + pageTitle: string | undefined +): string { + const meta = resolvePageMeta(pathname); + const title = pageTitle?.trim(); + const prefix = meta.singular ?? meta.name; + + // Generic titles ("Runs", "Run") identify nothing on their own; prefer ids/filters from the URL + const isGenericTitle = + !title || + title.toLowerCase() === meta.name.toLowerCase() || + title.toLowerCase() === prefix.toLowerCase(); + + if (isGenericTitle) { + // Named entity from the URL (task/agent slug) is the label on its own; its icon carries the type + if (meta.entityName) return truncateLabel(meta.entityName); + + // The Tasks page filtered to a single task type takes that type as the whole name + if (meta.icon === "tasks") { + const types = new URLSearchParams(search).getAll("types"); + if (types.length === 1 && TASK_TYPE_LABELS[types[0]]) { + return TASK_TYPE_LABELS[types[0]]; + } + } + + const detailId = detailIdFromPath(pathname); + if (detailId) return `${prefix}: ${detailId}`; + + const filters = describeFilters(search); + return truncateLabel(filters ? `${meta.name}: ${filters}` : meta.name); + } + + const label = title.toLowerCase().startsWith(prefix.toLowerCase()) + ? title + : `${prefix}: ${title}`; + return truncateLabel(label); +} + +/** + * The user's favorited pages with any in-flight mutations applied, so the star button and the + * side menu section update instantly and stay in sync while the server round-trip completes. + */ +export function useFavorites(): FavoritePage[] { + const user = useOptionalUser(); + const fetchers = useFetchers(); + + let favorites = user?.dashboardPreferences.sideMenu?.favorites ?? []; + + for (const fetcher of fetchers) { + if (fetcher.formAction !== FAVORITES_ACTION_PATH || !fetcher.formData) continue; + + const intent = fetcher.formData.get("intent"); + const id = fetcher.formData.get("id"); + if (typeof id !== "string") continue; + + switch (intent) { + case "add": { + const url = fetcher.formData.get("url"); + const label = fetcher.formData.get("label"); + const icon = fetcher.formData.get("icon"); + if (typeof url !== "string" || typeof label !== "string") break; + if (!favorites.some((f) => f.url === url)) { + // Newest favorites go to the top of the section (matches addFavorite server-side) + favorites = [ + { id, url, label, icon: typeof icon === "string" ? icon : undefined }, + ...favorites, + ]; + } + break; + } + case "remove": { + favorites = favorites.filter((f) => f.id !== id); + break; + } + case "rename": { + const label = fetcher.formData.get("label"); + if (typeof label !== "string") break; + favorites = favorites.map((f) => (f.id === id ? { ...f, label } : f)); + break; + } + } + } + + return favorites; +} diff --git a/apps/webapp/app/components/navigation/sideMenuTypes.ts b/apps/webapp/app/components/navigation/sideMenuTypes.ts index bb245f4bcb3..508c4121175 100644 --- a/apps/webapp/app/components/navigation/sideMenuTypes.ts +++ b/apps/webapp/app/components/navigation/sideMenuTypes.ts @@ -2,6 +2,7 @@ import { z } from "zod"; // Valid section IDs that can have their collapsed state toggled export const SideMenuSectionIdSchema = z.enum([ + "favorites", "ai", "manage", "metrics", @@ -12,3 +13,58 @@ export const SideMenuSectionIdSchema = z.enum([ // Inferred type from the schema export type SideMenuSectionId = z.infer; + +// Size popover items to match the side-menu items, overriding the smaller small-menu-item +// defaults via tailwind-merge; icon carries the default dimmed color. +export const SIDE_MENU_POPOVER_ITEM_ICON = "h-5 w-5 text-text-dimmed"; +export const SIDE_MENU_POPOVER_ITEM_LABEL = "text-[0.90625rem] font-medium tracking-[-0.01em]"; + +/** Default top-to-bottom order of the customizable side menu sections. */ +export const DEFAULT_SECTION_ORDER: SideMenuSectionId[] = [ + "favorites", + "ai", + "metrics", + "deployments", + "manage", +]; + +/** + * Order entries by a saved preference. Entries missing from the saved order (e.g. a section or + * item that shipped after the user customized) are inserted at their default position relative + * to the entries around them, not dumped at the end — so "Favorites" still lands above "AI" for + * users who saved an order before favorites existed. + */ +export function orderByPreference( + entries: T[], + savedOrder: string[] | undefined +): T[] { + if (!savedOrder || savedOrder.length === 0) return entries; + + const defaultIndex = new Map(entries.map((entry, index) => [entry.id, index])); + // Set-dedupe: a corrupted saved order with duplicate ids must not render an entry twice + const orderedIds = [...new Set(savedOrder.filter((id) => defaultIndex.has(id)))]; + const missingIds = entries.map((entry) => entry.id).filter((id) => !orderedIds.includes(id)); + + for (const id of missingIds) { + const idDefault = defaultIndex.get(id) ?? 0; + let insertAt = orderedIds.length; + for (let i = 0; i < orderedIds.length; i++) { + if ((defaultIndex.get(orderedIds[i]) ?? 0) > idDefault) { + insertAt = i; + break; + } + } + orderedIds.splice(insertAt, 0, id); + } + + const byId = new Map(entries.map((entry) => [entry.id, entry])); + return orderedIds.map((id) => byId.get(id)!); +} + +/** Effective hidden state for a menu item: the user's override wins, else the item's default. */ +export function isItemHidden( + item: { id: string; defaultHidden?: boolean }, + hiddenItems: Record | undefined +): boolean { + return hiddenItems?.[item.id] ?? item.defaultHidden ?? false; +} diff --git a/apps/webapp/app/components/primitives/PageHeader.tsx b/apps/webapp/app/components/primitives/PageHeader.tsx index a7c68f059f5..691b9323bad 100644 --- a/apps/webapp/app/components/primitives/PageHeader.tsx +++ b/apps/webapp/app/components/primitives/PageHeader.tsx @@ -7,6 +7,7 @@ import { Header2 } from "./Headers"; import { LoadingBarDivider } from "./LoadingBarDivider"; import { SimpleTooltip } from "./Tooltip"; import { DashboardAgentLauncher } from "../dashboard-agent/dashboardAgentLauncher"; +import { FavoritePageButton } from "../navigation/FavoritePageButton"; type WithChildren = { children: React.ReactNode; @@ -46,8 +47,10 @@ type PageTitleProps = { }; export function PageTitle({ title, backButton, accessory }: PageTitleProps) { + const titleText = typeof title === "string" ? title : undefined; + return ( -
+
{backButton && (
)} {title} - {accessory !== undefined && - (typeof accessory === "string" ? ( - } - content={accessory} - className="max-w-xs" - disableHoverableContent - /> - ) : ( - accessory - ))} + {accessory !== undefined && ( + // ml-px optically evens the accessory against the title's tight text edge + + {typeof accessory === "string" ? ( + } + content={accessory} + className="max-w-xs" + disableHoverableContent + /> + ) : ( + accessory + )} + + )} + {/* -ml-1 pulls the star's button box near-flush: its inner padding then provides the + visual gap, matching the title-to-accessory spacing while hovered */} +
); } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.custom.$dashboardId/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.custom.$dashboardId/route.tsx index 8aa6427afbf..153004813a8 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.custom.$dashboardId/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.custom.$dashboardId/route.tsx @@ -48,7 +48,8 @@ import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { getTaskIdentifiers } from "~/models/task.server"; import { MetricDashboardPresenter } from "~/presenters/v3/MetricDashboardPresenter.server"; import { QueryPresenter } from "~/presenters/v3/QueryPresenter.server"; -import { requireUser, requireUserId } from "~/services/session.server"; +import { removeFavoritesByUrlSubstring } from "~/services/dashboardPreferences.server"; +import { requireUser } from "~/services/session.server"; import { EnvironmentParamSchema, queryPath, @@ -115,10 +116,10 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }; export const action = async ({ request, params }: ActionFunctionArgs) => { - const userId = await requireUserId(request); + const user = await requireUser(request); const { projectParam, organizationSlug, envParam, dashboardId } = ParamSchema.parse(params); - const project = await findProjectBySlug(organizationSlug, projectParam, userId); + const project = await findProjectBySlug(organizationSlug, projectParam, user.id); if (!project) { throw new Response("Project not found", { status: 404 }); } @@ -144,6 +145,12 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { where: { id: dashboard.id }, }); + // Drop any favorites pointing at this dashboard so the side menu doesn't keep a dead link + await removeFavoritesByUrlSubstring({ + user, + substring: `/dashboards/custom/${dashboard.friendlyId}`, + }); + return redirectWithSuccessMessage( v3BuiltInDashboardPath( { slug: organizationSlug }, diff --git a/apps/webapp/app/routes/resources.preferences.favorites.tsx b/apps/webapp/app/routes/resources.preferences.favorites.tsx new file mode 100644 index 00000000000..071d2644939 --- /dev/null +++ b/apps/webapp/app/routes/resources.preferences.favorites.tsx @@ -0,0 +1,80 @@ +import { json, type ActionFunctionArgs } from "@remix-run/node"; +import { z } from "zod"; +import { + addFavorite, + removeFavorite, + renameFavorite, +} from "~/services/dashboardPreferences.server"; +import { logger } from "~/services/logger.server"; +import { requireUser } from "~/services/session.server"; + +const FavoriteLabel = z + .string() + .transform((value) => value.trim()) + .pipe(z.string().min(1).max(64)); + +const RequestSchema = z.discriminatedUnion("intent", [ + z.object({ + intent: z.literal("add"), + id: z.string().min(1).max(64), + // App-relative URL only ("/..." but not protocol-relative "//...") + url: z + .string() + .min(1) + .max(2048) + .refine((url) => url.startsWith("/") && !url.startsWith("//"), { + message: "URL must be app-relative", + }), + label: FavoriteLabel, + icon: z + .string() + .regex(/^[a-z0-9-]+$/) + .max(64) + .optional(), + }), + z.object({ + intent: z.literal("remove"), + id: z.string().min(1).max(64), + }), + z.object({ + intent: z.literal("rename"), + id: z.string().min(1).max(64), + label: FavoriteLabel, + }), +]); + +export async function action({ request }: ActionFunctionArgs) { + const user = await requireUser(request); + + const formData = await request.formData(); + const result = RequestSchema.safeParse(Object.fromEntries(formData)); + + if (!result.success) { + return json({ success: false, error: "Invalid request data" }, { status: 400 }); + } + + // Errors come back as a response (never a throw, which would escalate a preferences write to + // the error boundary); the side menu's optimistic entries revert when the fetcher settles. + try { + switch (result.data.intent) { + case "add": { + const { id, url, label, icon } = result.data; + await addFavorite({ user, favorite: { id, url, label, icon } }); + break; + } + case "remove": { + await removeFavorite({ user, id: result.data.id }); + break; + } + case "rename": { + await renameFavorite({ user, id: result.data.id, label: result.data.label }); + break; + } + } + } catch (error) { + logger.error("Failed to update favorites", { error: String(error) }); + return json({ success: false, error: "Failed to save preferences" }, { status: 500 }); + } + + return json({ success: true }); +} diff --git a/apps/webapp/app/routes/resources.preferences.sidemenu.tsx b/apps/webapp/app/routes/resources.preferences.sidemenu.tsx index 4e43269d0ea..35b11a4cf1c 100644 --- a/apps/webapp/app/routes/resources.preferences.sidemenu.tsx +++ b/apps/webapp/app/routes/resources.preferences.sidemenu.tsx @@ -4,7 +4,12 @@ import { SideMenuSectionIdSchema, type SideMenuSectionId, } from "~/components/navigation/sideMenuTypes"; -import { updateItemOrder, updateSideMenuPreferences } from "~/services/dashboardPreferences.server"; +import { + updateItemOrder, + updateSideMenuCustomization, + updateSideMenuPreferences, +} from "~/services/dashboardPreferences.server"; +import { logger } from "~/services/logger.server"; import { requireUser } from "~/services/session.server"; // Transforms form data string "true"/"false" to boolean, or undefined if not present @@ -22,11 +27,32 @@ const RequestSchema = z.object({ organizationId: z.string().optional(), listId: z.string().optional(), itemOrder: z.string().optional(), // JSON-encoded string[] + customization: z.string().optional(), // JSON-encoded CustomizationSchema +}); + +// Payload of the "Customize sidebar" modal. For the nullable fields, null resets to default and +// an absent field leaves the stored value unchanged. +const CustomizationSchema = z.object({ + sectionOrder: z.array(z.string().max(64)).max(50).nullish(), + hiddenItems: z.record(z.string().max(64), z.boolean()).nullish(), + sectionItemOrder: z.record(z.string().max(64), z.array(z.string().max(64)).max(100)).nullish(), + favorites: z + .array(z.object({ id: z.string().max(64), label: z.string().max(64) })) + .max(100) + .optional(), + removedFavoriteIds: z.array(z.string().max(64)).max(100).optional(), }); export async function action({ request }: ActionFunctionArgs) { const user = await requireUser(request); + // Every writer below deliberately skips impersonated sessions so an admin's browsing can't + // rewrite the customer's saved layout. That skip is a no-op, not a failed save, so report it as + // success: it must not reach the callers that surface write failures to the user. + if (user.isImpersonating) { + return json({ success: true }); + } + const formData = await request.formData(); const rawData = Object.fromEntries(formData); @@ -35,6 +61,42 @@ export async function action({ request }: ActionFunctionArgs) { return json({ success: false, error: "Invalid request data" }, { status: 400 }); } + // Handle a "Customize sidebar" modal submit + if (result.data.customization) { + let parsed: unknown; + try { + parsed = JSON.parse(result.data.customization); + } catch { + parsed = null; + } + const customizationResult = CustomizationSchema.safeParse(parsed); + if (!customizationResult.success) { + return json({ success: false, error: "Invalid request data" }, { status: 400 }); + } + const { sectionOrder, hiddenItems, sectionItemOrder, favorites, removedFavoriteIds } = + customizationResult.data; + // The modal keeps its "Confirm" pending until this responds, so failures must come back as a + // response (never a throw, which would escalate a preferences write to the error boundary). + try { + const updated = await updateSideMenuCustomization({ + user, + sectionOrder, + hiddenItems, + sectionItemOrder, + favorites, + removedFavoriteIds, + }); + // undefined means nothing was written (impersonating, or the user row is gone) + if (!updated) { + return json({ success: false, error: "Failed to save preferences" }, { status: 500 }); + } + } catch (error) { + logger.error("Failed to save sidebar customization", { error: String(error) }); + return json({ success: false, error: "Failed to save preferences" }, { status: 500 }); + } + return json({ success: true }); + } + // Handle item order update if (result.data.organizationId && result.data.listId && result.data.itemOrder) { let parsed: unknown; diff --git a/apps/webapp/app/services/dashboardPreferences.server.ts b/apps/webapp/app/services/dashboardPreferences.server.ts index a8b9149eff5..9f5f84fe2e8 100644 --- a/apps/webapp/app/services/dashboardPreferences.server.ts +++ b/apps/webapp/app/services/dashboardPreferences.server.ts @@ -1,8 +1,21 @@ import { z } from "zod"; -import { prisma } from "~/db.server"; +import { $transaction, prisma } from "~/db.server"; import { logger } from "./logger.server"; import { type UserFromSession } from "./session.server"; +const FavoritePage = z.object({ + /** Stable id, generated client-side when the page is favorited. */ + id: z.string(), + /** App-relative URL including any search params (filters, tabs). */ + url: z.string(), + /** Display label shown in the side menu; user-renamable. */ + label: z.string(), + /** Key into the favorite page icon registry. */ + icon: z.string().optional(), +}); + +export type FavoritePage = z.infer; + const SideMenuPreferences = z.object({ isCollapsed: z.boolean().default(false), /** Expanded side menu width in px, set by the resize handle. */ @@ -18,6 +31,14 @@ const SideMenuPreferences = z.object({ }) ) .optional(), + /** Pages the user favorited, in display order. */ + favorites: z.array(FavoritePage).optional(), + /** Custom top-to-bottom order of side menu sections (section ids). */ + sectionOrder: z.array(z.string()).optional(), + /** Per-item visibility overrides (item id -> hidden). Items absent fall back to their default. */ + hiddenItems: z.record(z.string(), z.boolean()).optional(), + /** Custom item order within a section (section id -> item ids). */ + sectionItemOrder: z.record(z.string(), z.array(z.string())).optional(), }); export type SideMenuPreferences = z.infer; @@ -59,6 +80,51 @@ export function getDashboardPreferences(data?: any | null): DashboardPreferences return result.data; } +/** + * Every preference writer is a read-modify-write over one JSON column, and several fire + * concurrently (debounced collapse/width, favorite toggles, the customize modal, dashboard + * reorders). Each write re-reads the row under a FOR UPDATE lock so concurrent writers + * serialize instead of clobbering each other's fields with stale reads — without the lock, a + * debounced collapse write could resurrect customizations the modal's Reset just cleared. + * + * Return undefined from `mutate` to skip the write (no-op update). + */ +async function mutateDashboardPreferences( + userId: string, + mutate: (current: DashboardPreferences) => DashboardPreferences | undefined +) { + return await $transaction( + prisma, + "mutateDashboardPreferences", + async (tx) => { + const rows = await tx.$queryRaw>` + SELECT "dashboardPreferences" FROM "User" WHERE id = ${userId} FOR UPDATE + `; + if (rows.length === 0) { + return undefined; + } + + const updated = mutate(getDashboardPreferences(rows[0].dashboardPreferences)); + if (!updated) { + return undefined; + } + + return await tx.user.update({ + where: { + id: userId, + }, + data: { + dashboardPreferences: updated, + }, + }); + }, + // Concurrent writers queue on the row lock, so under load (several debounced writes plus a + // revalidation burst) a transaction can time out acquiring a connection or the lock; those + // codes are retriable and preference writes are idempotent. + { maxRetries: 3 } + ); +} + export async function updateCurrentProjectEnvironmentId({ user, projectId, @@ -72,7 +138,9 @@ export async function updateCurrentProjectEnvironmentId({ return; } - //only update if the existing preferences are different + // Fast path: this runs on nearly every navigation (env layout loader), so skip the locked + // transaction when the session snapshot already matches. The in-transaction check below stays + // authoritative for the rare stale-snapshot case. if ( user.dashboardPreferences.currentProjectId === projectId && user.dashboardPreferences.projects[projectId]?.currentEnvironment?.id === environmentId @@ -80,26 +148,26 @@ export async function updateCurrentProjectEnvironmentId({ return; } - //ok we need to update the preferences - const updatedPreferences: DashboardPreferences = { - ...user.dashboardPreferences, - currentProjectId: projectId, - projects: { - ...user.dashboardPreferences.projects, - [projectId]: { - ...user.dashboardPreferences.projects[projectId], - currentEnvironment: { id: environmentId }, - }, - }, - }; + return mutateDashboardPreferences(user.id, (prefs) => { + //only update if the existing preferences are different + if ( + prefs.currentProjectId === projectId && + prefs.projects[projectId]?.currentEnvironment?.id === environmentId + ) { + return undefined; + } - return prisma.user.update({ - where: { - id: user.id, - }, - data: { - dashboardPreferences: updatedPreferences, - }, + return { + ...prefs, + currentProjectId: projectId, + projects: { + ...prefs.projects, + [projectId]: { + ...prefs.projects[projectId], + currentEnvironment: { id: environmentId }, + }, + }, + }; }); } @@ -108,19 +176,10 @@ export async function clearCurrentProject({ user }: { user: UserFromSession }) { return; } - const updatedPreferences: DashboardPreferences = { - ...user.dashboardPreferences, + return mutateDashboardPreferences(user.id, (prefs) => ({ + ...prefs, currentProjectId: undefined, - }; - - return prisma.user.update({ - where: { - id: user.id, - }, - data: { - dashboardPreferences: updatedPreferences, - }, - }); + })); } export async function updateSideMenuPreferences({ @@ -140,48 +199,234 @@ export async function updateSideMenuPreferences({ return; } - // Parse with schema to apply defaults, then overlay any new values - const currentSideMenu = SideMenuPreferences.parse(user.dashboardPreferences.sideMenu ?? {}); + return mutateDashboardPreferences(user.id, (prefs) => { + // Parse with schema to apply defaults, then overlay any new values + const currentSideMenu = SideMenuPreferences.parse(prefs.sideMenu ?? {}); + + // Build the updated collapsedSections map + let updatedCollapsedSections = { ...currentSideMenu.collapsedSections }; + + if (sectionCollapsed) { + updatedCollapsedSections[sectionCollapsed.sectionId] = sectionCollapsed.collapsed; + } + + const updatedSideMenu = SideMenuPreferences.parse({ + ...currentSideMenu, + ...(isCollapsed !== undefined && { isCollapsed }), + ...(width !== undefined && { width }), + collapsedSections: updatedCollapsedSections, + }); + + // Only update if something changed + const hasCollapsedSectionsChanged = + JSON.stringify(updatedSideMenu.collapsedSections) !== + JSON.stringify(currentSideMenu.collapsedSections); + + if ( + updatedSideMenu.isCollapsed === currentSideMenu.isCollapsed && + updatedSideMenu.width === currentSideMenu.width && + !hasCollapsedSectionsChanged + ) { + return undefined; + } - // Build the updated collapsedSections map - let updatedCollapsedSections = { ...currentSideMenu.collapsedSections }; + return { ...prefs, sideMenu: updatedSideMenu }; + }); +} - if (sectionCollapsed) { - updatedCollapsedSections[sectionCollapsed.sectionId] = sectionCollapsed.collapsed; +/** The most favorites a user can save; a sanity cap, not a product limit. */ +const MAX_FAVORITES = 50; + +export async function addFavorite({ + user, + favorite, +}: { + user: UserFromSession; + favorite: FavoritePage; +}) { + if (user.isImpersonating) { + return; } - const updatedSideMenu = SideMenuPreferences.parse({ - ...currentSideMenu, - ...(isCollapsed !== undefined && { isCollapsed }), - ...(width !== undefined && { width }), - collapsedSections: updatedCollapsedSections, + return mutateDashboardPreferences(user.id, (prefs) => { + const currentSideMenu = SideMenuPreferences.parse(prefs.sideMenu ?? {}); + const favorites = currentSideMenu.favorites ?? []; + + // The star is a toggle keyed on the exact URL, so an existing entry means we're already done + if (favorites.some((f) => f.url === favorite.url)) { + return undefined; + } + + if (favorites.length >= MAX_FAVORITES) { + return undefined; + } + + // Newest favorites go to the top of the section + return { + ...prefs, + sideMenu: { ...currentSideMenu, favorites: [favorite, ...favorites] }, + }; }); +} - // Only update if something changed - const hasCollapsedSectionsChanged = - JSON.stringify(updatedSideMenu.collapsedSections) !== - JSON.stringify(currentSideMenu.collapsedSections); +export async function removeFavorite({ user, id }: { user: UserFromSession; id: string }) { + if (user.isImpersonating) { + return; + } - if ( - updatedSideMenu.isCollapsed === currentSideMenu.isCollapsed && - updatedSideMenu.width === currentSideMenu.width && - !hasCollapsedSectionsChanged - ) { + return mutateDashboardPreferences(user.id, (prefs) => { + const currentSideMenu = SideMenuPreferences.parse(prefs.sideMenu ?? {}); + const favorites = currentSideMenu.favorites ?? []; + const remaining = favorites.filter((f) => f.id !== id); + + if (remaining.length === favorites.length) { + return undefined; + } + + return { + ...prefs, + sideMenu: { + ...currentSideMenu, + favorites: remaining.length > 0 ? remaining : undefined, + }, + }; + }); +} + +/** + * Remove any favorites whose URL contains the given substring. Used when the favorited entity + * itself is deleted (e.g. a custom dashboard's friendly id) so the side menu doesn't keep a + * dead link. + */ +export async function removeFavoritesByUrlSubstring({ + user, + substring, +}: { + user: UserFromSession; + substring: string; +}) { + if (user.isImpersonating) { return; } - const updatedPreferences: DashboardPreferences = { - ...user.dashboardPreferences, - sideMenu: updatedSideMenu, - }; + return mutateDashboardPreferences(user.id, (prefs) => { + const currentSideMenu = SideMenuPreferences.parse(prefs.sideMenu ?? {}); + const favorites = currentSideMenu.favorites ?? []; + const remaining = favorites.filter((favorite) => !favorite.url.includes(substring)); - return prisma.user.update({ - where: { - id: user.id, - }, - data: { - dashboardPreferences: updatedPreferences, - }, + if (remaining.length === favorites.length) { + return undefined; + } + + return { + ...prefs, + sideMenu: { + ...currentSideMenu, + favorites: remaining.length > 0 ? remaining : undefined, + }, + }; + }); +} + +export async function renameFavorite({ + user, + id, + label, +}: { + user: UserFromSession; + id: string; + label: string; +}) { + if (user.isImpersonating) { + return; + } + + return mutateDashboardPreferences(user.id, (prefs) => { + const currentSideMenu = SideMenuPreferences.parse(prefs.sideMenu ?? {}); + const favorites = currentSideMenu.favorites ?? []; + + const favorite = favorites.find((f) => f.id === id); + if (!favorite || favorite.label === label) { + return undefined; + } + + return { + ...prefs, + sideMenu: { + ...currentSideMenu, + favorites: favorites.map((f) => (f.id === id ? { ...f, label } : f)), + }, + }; + }); +} + +export async function updateSideMenuCustomization({ + user, + sectionOrder, + hiddenItems, + sectionItemOrder, + favorites, + removedFavoriteIds, +}: { + user: UserFromSession; + /** undefined = leave unchanged, null = reset to default */ + sectionOrder?: string[] | null; + /** undefined = leave unchanged, null = reset to default */ + hiddenItems?: Record | null; + /** undefined = leave unchanged, null = reset to default */ + sectionItemOrder?: Record | null; + /** Full favorites arrangement: new order + labels. undefined = leave unchanged. */ + favorites?: Array<{ id: string; label: string }>; + /** Favorites deleted from the customize modal. */ + removedFavoriteIds?: string[]; +}) { + if (user.isImpersonating) { + return; + } + + return mutateDashboardPreferences(user.id, (prefs) => { + const currentSideMenu = SideMenuPreferences.parse(prefs.sideMenu ?? {}); + const next: SideMenuPreferences = { ...currentSideMenu }; + + if (sectionOrder !== undefined) { + next.sectionOrder = sectionOrder && sectionOrder.length > 0 ? sectionOrder : undefined; + } + + if (hiddenItems !== undefined) { + next.hiddenItems = + hiddenItems && Object.keys(hiddenItems).length > 0 ? hiddenItems : undefined; + } + + if (sectionItemOrder !== undefined) { + next.sectionItemOrder = + sectionItemOrder && Object.keys(sectionItemOrder).length > 0 ? sectionItemOrder : undefined; + } + + if (favorites !== undefined || removedFavoriteIds !== undefined) { + const removed = new Set(removedFavoriteIds ?? []); + const current = (currentSideMenu.favorites ?? []).filter((f) => !removed.has(f.id)); + const byId = new Map(current.map((f) => [f.id, f])); + const rearranged: FavoritePage[] = []; + + for (const { id, label } of favorites ?? []) { + const existing = byId.get(id); + if (!existing) continue; + const trimmed = label.trim(); + rearranged.push({ ...existing, label: trimmed.length > 0 ? trimmed : existing.label }); + byId.delete(id); + } + + // Favorites the payload didn't mention (e.g. added mid-edit) keep their place at the end + for (const favorite of current) { + if (byId.has(favorite.id)) { + rearranged.push(favorite); + } + } + + next.favorites = rearranged.length > 0 ? rearranged : undefined; + } + + return { ...prefs, sideMenu: SideMenuPreferences.parse(next) }; }); } @@ -209,34 +454,24 @@ export async function updateItemOrder({ return; } - const currentSideMenu = SideMenuPreferences.parse(user.dashboardPreferences.sideMenu ?? {}); - const currentOrg = currentSideMenu.organizations?.[organizationId]; - - const updatedSideMenu = SideMenuPreferences.parse({ - ...currentSideMenu, - organizations: { - ...currentSideMenu.organizations, - [organizationId]: { - ...currentOrg, - orderedItems: { - ...currentOrg?.orderedItems, - [listId]: order, + return mutateDashboardPreferences(user.id, (prefs) => { + const currentSideMenu = SideMenuPreferences.parse(prefs.sideMenu ?? {}); + const currentOrg = currentSideMenu.organizations?.[organizationId]; + + const updatedSideMenu = SideMenuPreferences.parse({ + ...currentSideMenu, + organizations: { + ...currentSideMenu.organizations, + [organizationId]: { + ...currentOrg, + orderedItems: { + ...currentOrg?.orderedItems, + [listId]: order, + }, }, }, - }, - }); + }); - const updatedPreferences: DashboardPreferences = { - ...user.dashboardPreferences, - sideMenu: updatedSideMenu, - }; - - return prisma.user.update({ - where: { - id: user.id, - }, - data: { - dashboardPreferences: updatedPreferences, - }, + return { ...prefs, sideMenu: updatedSideMenu }; }); } From 73eb4c5c1623717502622c135de9004f97b7e6c9 Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Mon, 27 Jul 2026 16:34:44 +0100 Subject: [PATCH 18/67] feat(webapp): Improve the Integrations page layout (#4379) ## Summary The project Integrations page now uses the same settings layout as the org SSO page: a centered column of titled rows with dividers, instead of headings over bordered boxes. GitHub, Vercel and build settings read as one consistent list, and the page titles itself "Integrations". Confirmations persist rather than vanishing once you move past them (`GitHub app: Installed`, `Vercel project: Connected`), plan-gated rows offer an Upgrade button instead of a dead toggle, a disabled toggle explains why in place and highlights the control that unlocks it, and warnings are rows with a hazard icon and their recovery action on the right. Copy throughout leads with the outcome instead of restating the field label. Two fixes along the way: a nested `
` in the Vercel panel that failed hydration and silently truncated the page, and every settings row carrying a few pixels more space above its title than below its description. ### Before CleanShot 2026-07-26 at 21 56 42@2x ### After CleanShot 2026-07-26 at 19 14 28@2x --- .../app/assets/icons/PadlockRoundedIcon.tsx | 34 + .../app/components/BlankStatePanels.tsx | 14 +- .../integrations/VercelBuildSettings.tsx | 507 +++++++++----- .../integrations/VercelOnboardingModal.tsx | 72 +- .../app/components/primitives/Buttons.tsx | 11 + .../app/components/primitives/Select.tsx | 7 +- .../components/primitives/SettingsLayout.tsx | 42 +- .../route.tsx | 339 +++++----- .../route.tsx | 16 +- ...cts.$projectParam.env.$envParam.github.tsx | 515 ++++++++++----- ...cts.$projectParam.env.$envParam.vercel.tsx | 618 +++++++++--------- .../app/routes/storybook.buttons/route.tsx | 12 + .../app/routes/storybook.icons/route.tsx | 2 + apps/webapp/app/tailwind.css | 10 + 14 files changed, 1351 insertions(+), 848 deletions(-) create mode 100644 apps/webapp/app/assets/icons/PadlockRoundedIcon.tsx diff --git a/apps/webapp/app/assets/icons/PadlockRoundedIcon.tsx b/apps/webapp/app/assets/icons/PadlockRoundedIcon.tsx new file mode 100644 index 00000000000..56f2543fc3f --- /dev/null +++ b/apps/webapp/app/assets/icons/PadlockRoundedIcon.tsx @@ -0,0 +1,34 @@ +export function PadlockRoundedIcon({ className }: { className?: string }) { + return ( + + + + + + ); +} diff --git a/apps/webapp/app/components/BlankStatePanels.tsx b/apps/webapp/app/components/BlankStatePanels.tsx index 5d166158e0b..5fde85cc050 100644 --- a/apps/webapp/app/components/BlankStatePanels.tsx +++ b/apps/webapp/app/components/BlankStatePanels.tsx @@ -701,14 +701,12 @@ function DeploymentOnboardingSteps() { Deploy automatically with every push. Read the{" "} full guide. -
- -
+ diff --git a/apps/webapp/app/components/integrations/VercelBuildSettings.tsx b/apps/webapp/app/components/integrations/VercelBuildSettings.tsx index 1abd54b0121..fc45d2f6019 100644 --- a/apps/webapp/app/components/integrations/VercelBuildSettings.tsx +++ b/apps/webapp/app/components/integrations/VercelBuildSettings.tsx @@ -1,5 +1,12 @@ import { Switch } from "~/components/primitives/Switch"; +import { LinkButton } from "~/components/primitives/Buttons"; import { Label } from "~/components/primitives/Label"; +import { + SettingsRow, + SettingsRowDescription, + SettingsRowTitle, +} from "~/components/primitives/SettingsLayout"; +import { cn } from "~/utils/cn"; import { Hint } from "~/components/primitives/Hint"; import { TextLink } from "~/components/primitives/TextLink"; import { SimpleTooltip } from "~/components/primitives/Tooltip"; @@ -31,6 +38,7 @@ type BuildSettingsFieldsProps = { currentTriggerVersionFetchFailed?: boolean; /** Hide the section-level master toggles for "Pull env vars" and "Discover new env vars". */ hideSectionToggles?: boolean; + layout?: "settings" | "card"; }; export function BuildSettingsFields({ @@ -48,187 +56,326 @@ export function BuildSettingsFields({ currentTriggerVersion, currentTriggerVersionFetchFailed, hideSectionToggles, + layout = "card", }: BuildSettingsFieldsProps) { const isSlugDisabled = (slug: EnvSlug) => !!disabledEnvSlugs?.[slug]; const enabledSlugs = availableEnvSlugs.filter((s) => !isSlugDisabled(s)); + const envVarSections = + layout === "settings" ? ( + <> + + Configure env vars + + ) : undefined + } + /> + {availableEnvSlugs.map((slug) => ( + { + onPullEnvVarsChange( + checked + ? [...pullEnvVarsBeforeBuild, slug] + : pullEnvVarsBeforeBuild.filter((s) => s !== slug) + ); + }} + /> + ))} + + + {availableEnvSlugs.map((slug) => { + const pullOff = !pullEnvVarsBeforeBuild.includes(slug); + return ( + { + onDiscoverEnvVarsChange( + checked ? [...discoverEnvVars, slug] : discoverEnvVars.filter((s) => s !== slug) + ); + }} + /> + ); + })} + + ) : null; + + const atomicSections = + layout === "settings" ? ( + <> + { + onAtomicBuildsChange(checked ? ["prod"] : []); + }} + /> + } + > +
+ Atomic deployments + + Promotes your Vercel deployment and your tasks together in Production, so your app + never runs against a mismatched task version. Requires turning off "Auto-assign Custom + Production Domains" on your Vercel project, which Trigger.dev does for you.{" "} + + Learn more + + . + + {currentTriggerVersion && ( + + Currently pinned to{" "} + {currentTriggerVersion} in + Vercel production. + + )} + {!currentTriggerVersion && currentTriggerVersionFetchFailed && ( + + Couldn't read TRIGGER_VERSION{" "} + from Vercel. Check the Vercel dashboard to confirm the production pin. + + )} +
+ + + {atomicBuilds.includes("prod") && onAutoPromoteChange !== undefined && ( + + } + /> + )} + + ) : null; + return ( <> + {envVarSections} {/* Pull env vars before build */} -
-
-
- - {!hideSectionToggles && availableEnvSlugs.length > 1 && ( - 0 && - enabledSlugs.every((s) => pullEnvVarsBeforeBuild.includes(s)) - } - onCheckedChange={(checked) => { - onPullEnvVarsChange(checked ? [...enabledSlugs] : []); - }} - /> - )} -
- - Select which environments should pull environment variables from Vercel before each - build.{" "} - {envVarsConfigLink && ( - <> - Configure which variables to pull. - - )} - -
-
- {availableEnvSlugs.map((slug) => { - const envType = envSlugToType(slug); - const disabled = isSlugDisabled(slug); - const disabledReason = disabledEnvSlugs?.[slug]; - const row = ( -
-
- - - {environmentFullTitle({ type: envType })} - -
+ {layout === "card" && ( +
+
+
+ + {!hideSectionToggles && availableEnvSlugs.length > 1 && ( 0 && + enabledSlugs.every((s) => pullEnvVarsBeforeBuild.includes(s)) + } onCheckedChange={(checked) => { - onPullEnvVarsChange( - checked - ? [...pullEnvVarsBeforeBuild, slug] - : pullEnvVarsBeforeBuild.filter((s) => s !== slug) - ); + onPullEnvVarsChange(checked ? [...enabledSlugs] : []); }} /> -
- ); - if (disabled && disabledReason) { - return ; - } - return row; - })} + )} +
+ + Select which environments should pull environment variables from Vercel before each + build.{" "} + {envVarsConfigLink && ( + <> + Configure which variables to pull. + + )} + +
+
+ {availableEnvSlugs.map((slug) => { + const envType = envSlugToType(slug); + const disabled = isSlugDisabled(slug); + const disabledReason = disabledEnvSlugs?.[slug]; + const row = ( +
+
+ + + {environmentFullTitle({ type: envType })} + +
+ { + onPullEnvVarsChange( + checked + ? [...pullEnvVarsBeforeBuild, slug] + : pullEnvVarsBeforeBuild.filter((s) => s !== slug) + ); + }} + /> +
+ ); + if (disabled && disabledReason) { + return ( + + ); + } + return row; + })} +
-
+ )} {/* Discover new env vars */} -
-
-
- - {!hideSectionToggles && availableEnvSlugs.length > 1 && ( - 0 && - enabledSlugs.every( - (s) => discoverEnvVars.includes(s) || !pullEnvVarsBeforeBuild.includes(s) - ) && - enabledSlugs.some((s) => discoverEnvVars.includes(s)) - } - disabled={!enabledSlugs.some((s) => pullEnvVarsBeforeBuild.includes(s))} - onCheckedChange={(checked) => { - onDiscoverEnvVarsChange( - checked ? enabledSlugs.filter((s) => pullEnvVarsBeforeBuild.includes(s)) : [] - ); - }} - /> - )} -
- - Select which environments should automatically discover and create new environment - variables from Vercel during builds. - -
-
- {availableEnvSlugs.map((slug) => { - const envType = envSlugToType(slug); - const disabled = isSlugDisabled(slug); - const disabledReason = disabledEnvSlugs?.[slug]; - const isPullDisabled = !pullEnvVarsBeforeBuild.includes(slug); - const row = ( -
-
- - - {environmentFullTitle({ type: envType })} - -
+ {layout === "card" && ( +
+
+
+ + {!hideSectionToggles && availableEnvSlugs.length > 1 && ( 0 && + enabledSlugs.every( + (s) => discoverEnvVars.includes(s) || !pullEnvVarsBeforeBuild.includes(s) + ) && + enabledSlugs.some((s) => discoverEnvVars.includes(s)) + } + disabled={!enabledSlugs.some((s) => pullEnvVarsBeforeBuild.includes(s))} onCheckedChange={(checked) => { onDiscoverEnvVarsChange( - checked - ? [...discoverEnvVars, slug] - : discoverEnvVars.filter((s) => s !== slug) + checked ? enabledSlugs.filter((s) => pullEnvVarsBeforeBuild.includes(s)) : [] ); }} /> -
- ); - if (disabled && disabledReason) { - return ; - } - return row; - })} + )} +
+ + Select which environments should automatically discover and create new environment + variables from Vercel during builds. + +
+
+ {availableEnvSlugs.map((slug) => { + const envType = envSlugToType(slug); + const disabled = isSlugDisabled(slug); + const disabledReason = disabledEnvSlugs?.[slug]; + const isPullDisabled = !pullEnvVarsBeforeBuild.includes(slug); + const row = ( +
+
+ + + {environmentFullTitle({ type: envType })} + +
+ { + onDiscoverEnvVarsChange( + checked + ? [...discoverEnvVars, slug] + : discoverEnvVars.filter((s) => s !== slug) + ); + }} + /> +
+ ); + if (disabled && disabledReason) { + return ( + + ); + } + return row; + })} +
-
+ )} + + {atomicSections} {/* Atomic deployments */} -
-
- - { - onAtomicBuildsChange(checked ? ["prod"] : []); - }} - /> -
- - When enabled, production deployments wait for Vercel deployment to complete before - promoting the Trigger.dev deployment. This will disable the "Auto-assign Custom Production - Domains" option in your Vercel project settings to perform staged deployments.{" "} - - Learn more - - . - - {currentTriggerVersion && ( + {layout === "card" && ( +
+
+ + { + onAtomicBuildsChange(checked ? ["prod"] : []); + }} + /> +
- Currently pinned to{" "} - {currentTriggerVersion} in Vercel - production. + When enabled, production deployments wait for Vercel deployment to complete before + promoting the Trigger.dev deployment. This will disable the "Auto-assign Custom + Production Domains" option in your Vercel project settings to perform staged + deployments.{" "} + + Learn more + + . - )} - {!currentTriggerVersion && currentTriggerVersionFetchFailed && ( - - Couldn't read TRIGGER_VERSION from - Vercel — check the Vercel dashboard to confirm the production pin. - - )} -
+ {currentTriggerVersion && ( + + Currently pinned to{" "} + {currentTriggerVersion} in Vercel + production. + + )} + {!currentTriggerVersion && currentTriggerVersionFetchFailed && ( + + Couldn't read TRIGGER_VERSION from + Vercel — check the Vercel dashboard to confirm the production pin. + + )} +
+ )} {/* Auto promotion — only visible when atomic deployments are on */} - {atomicBuilds.includes("prod") && onAutoPromoteChange !== undefined && ( + {layout === "card" && atomicBuilds.includes("prod") && onAutoPromoteChange !== undefined && (
@@ -248,3 +395,59 @@ export function BuildSettingsFields({ ); } + +function EnvToggleRow({ + slug, + checked, + disabled, + disabledReason, + unlockHint, + unlockTarget, + onCheckedChange, +}: { + slug: EnvSlug; + checked: boolean; + disabled: boolean; + disabledReason?: string; + unlockHint?: string; + unlockTarget?: string; + onCheckedChange: (checked: boolean) => void; +}) { + const envType = envSlugToType(slug); + + return ( + + + + } + > +
+
+ + + {environmentFullTitle({ type: envType })} + +
+ {disabled && disabledReason ? ( + {disabledReason} + ) : null} +
+ + ); +} diff --git a/apps/webapp/app/components/integrations/VercelOnboardingModal.tsx b/apps/webapp/app/components/integrations/VercelOnboardingModal.tsx index 85c7816b73e..cd5f9eee505 100644 --- a/apps/webapp/app/components/integrations/VercelOnboardingModal.tsx +++ b/apps/webapp/app/components/integrations/VercelOnboardingModal.tsx @@ -13,6 +13,7 @@ import { FormButtons } from "~/components/primitives/FormButtons"; import { FormError } from "~/components/primitives/FormError"; import { Header3 } from "~/components/primitives/Headers"; import { Hint } from "~/components/primitives/Hint"; +import { InputGroup } from "~/components/primitives/InputGroup"; import { Label } from "~/components/primitives/Label"; import { Paragraph } from "~/components/primitives/Paragraph"; import { Select, SelectItem } from "~/components/primitives/Select"; @@ -797,10 +798,8 @@ export function VercelOnboardingModal({
{showProjectSelection && (
- Select Vercel Project - - Choose which Vercel project to connect with this Trigger.dev project. Your API keys - will be automatically synced to Vercel. + + Choose the Vercel project to pair with this Trigger.dev project. {availableProjects.length === 0 ? ( @@ -808,41 +807,42 @@ export function VercelOnboardingModal({ No Vercel projects found. Please create a project in Vercel first. ) : ( - + + + + Your{" "} + + TRIGGER_SECRET_KEY + {" "} + is synced to Vercel for each environment once connected. + + )} {projectSelectionError && {projectSelectionError}} - - Once connected, your{" "} - - TRIGGER_SECRET_KEY - {" "} - will be automatically synced to Vercel for each environment. - - @@ -866,7 +866,7 @@ export function VercelOnboardingModal({
} cancelButton={ - } diff --git a/apps/webapp/app/components/primitives/Buttons.tsx b/apps/webapp/app/components/primitives/Buttons.tsx index 323f3743826..6075d52f1bf 100644 --- a/apps/webapp/app/components/primitives/Buttons.tsx +++ b/apps/webapp/app/components/primitives/Buttons.tsx @@ -89,6 +89,13 @@ const theme = { shortcut: "border-text-bright text-text-bright group-hover/button:border-text-bright/60", icon: "text-text-bright", }, + warning: { + textColor: "text-warning transition group-disabled/button:text-warning/60", + button: + "bg-warning/10 border border-warning/20 group-hover/button:bg-warning/20 group-hover/button:border-warning/40 group-disabled/button:opacity-60 group-disabled/button:pointer-events-none", + shortcut: "border-warning/40 text-warning group-hover/button:border-warning/60", + icon: "text-warning", + }, docs: { textColor: "text-blue-200/70 transition group-disabled/button:text-text-dimmed/80", button: @@ -133,6 +140,10 @@ const variant = { "danger/medium": createVariant("medium", "danger"), "danger/large": createVariant("large", "danger"), "danger/extra-large": createVariant("extra-large", "danger"), + "warning/small": createVariant("small", "warning"), + "warning/medium": createVariant("medium", "warning"), + "warning/large": createVariant("large", "warning"), + "warning/extra-large": createVariant("extra-large", "warning"), "docs/small": createVariant("small", "docs"), "docs/medium": createVariant("medium", "docs"), "docs/large": createVariant("large", "docs"), diff --git a/apps/webapp/app/components/primitives/Select.tsx b/apps/webapp/app/components/primitives/Select.tsx index 66b291b7524..3c4c8faad0c 100644 --- a/apps/webapp/app/components/primitives/Select.tsx +++ b/apps/webapp/app/components/primitives/Select.tsx @@ -38,6 +38,9 @@ const variants = { "secondary/small": { button: cn(sizes.small.button, style.secondary.button), }, + "secondary/medium": { + button: cn(sizes.medium.button, style.secondary.button), + }, "tertiary/small": { button: cn(sizes.small.button, style.tertiary.button), }, @@ -692,11 +695,11 @@ export function ComboBox({ ...props }: ComboBoxProps) { return ( -
+
} - className="flex-1 bg-transparent text-xs text-text-dimmed outline-hidden" + className="flex-1 border-0 bg-transparent text-xs text-text-dimmed outline-hidden focus:border-0 focus:ring-0" {...props} /> {shortcut && ( diff --git a/apps/webapp/app/components/primitives/SettingsLayout.tsx b/apps/webapp/app/components/primitives/SettingsLayout.tsx index 7d6484fb467..dae340e9835 100644 --- a/apps/webapp/app/components/primitives/SettingsLayout.tsx +++ b/apps/webapp/app/components/primitives/SettingsLayout.tsx @@ -1,3 +1,4 @@ +import { ExclamationCircleIcon, ExclamationTriangleIcon } from "@heroicons/react/20/solid"; import { type ReactNode } from "react"; import { MainHorizontallyCenteredContainer } from "~/components/layout/AppLayout"; import { cn } from "~/utils/cn"; @@ -100,7 +101,10 @@ export function SettingsRowTitle({ htmlFor?: string; className?: string; }) { - const classes = cn("font-sans text-sm font-semibold leading-tight text-text-bright", className); + const classes = cn( + "block font-sans text-sm font-semibold leading-tight text-text-bright", + className + ); return htmlFor ? (
@@ -335,6 +341,12 @@ export default function Story() { /> This is a delete button +
diff --git a/apps/webapp/app/routes/storybook.icons/route.tsx b/apps/webapp/app/routes/storybook.icons/route.tsx index a465e05fb5a..36acd29b234 100644 --- a/apps/webapp/app/routes/storybook.icons/route.tsx +++ b/apps/webapp/app/routes/storybook.icons/route.tsx @@ -93,6 +93,7 @@ import { MoveUpIcon } from "~/assets/icons/MoveUpIcon"; import { NodejsLogoIcon } from "~/assets/icons/NodejsLogoIcon"; import { OneTreeIcon } from "~/assets/icons/OneTreeIcon"; import { PadlockIcon } from "~/assets/icons/PadlockIcon"; +import { PadlockRoundedIcon } from "~/assets/icons/PadlockRoundedIcon"; import { PauseIcon } from "~/assets/icons/PauseIcon"; import { PlaygroundIcon } from "~/assets/icons/PlaygroundIcon"; import { PlusIcon } from "~/assets/icons/PlusIcon"; @@ -233,6 +234,7 @@ const icons: IconEntry[] = [ { name: "OneTreeIcon", render: simple(OneTreeIcon) }, { name: "OpenAIIcon", render: simple(OpenAIIcon) }, { name: "PadlockIcon", render: simple(PadlockIcon) }, + { name: "PadlockRoundedIcon", render: simple(PadlockRoundedIcon) }, { name: "PauseIcon", render: simple(PauseIcon) }, { name: "PerplexityIcon", render: simple(PerplexityIcon) }, { name: "PlaygroundIcon", render: simple(PlaygroundIcon) }, diff --git a/apps/webapp/app/tailwind.css b/apps/webapp/app/tailwind.css index 2bc91d77323..ab845b63353 100644 --- a/apps/webapp/app/tailwind.css +++ b/apps/webapp/app/tailwind.css @@ -639,3 +639,13 @@ @apply leading-relaxed; } } + +form:has(.unlock-hint-staging-env:hover) [data-unlock-target="staging-env"], +form:has(.unlock-hint-pull-prod:hover) [data-unlock-target="pull-prod"], +form:has(.unlock-hint-pull-stg:hover) [data-unlock-target="pull-stg"], +form:has(.unlock-hint-pull-preview:hover) [data-unlock-target="pull-preview"], +form:has(.unlock-hint-pull-dev:hover) [data-unlock-target="pull-dev"] { + border-radius: 0.25rem; + outline: 2px dashed var(--color-warning); + outline-offset: 2px; +} From d189ce17d385b33ef44d384a9715b85ac75be139 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:40:31 +0100 Subject: [PATCH 19/67] chore: release v4.5.8 (#4364) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary 2 new features, 9 improvements, 3 bug fixes. ## Highlights - Allow additional environment API keys to create scoped public access tokens through the Trigger.dev API. Use server-issued public access tokens for batch operations so environment-scoped API keys can read batch results. ([#4387](https://github.com/triggerdotdev/trigger.dev/pull/4387)) ## Improvements - Preserve the partial assistant message when a chat turn's model stream fails mid-response. `chat.agent` now passes the recovered partial to `onTurnComplete`, and `chat.createSession`'s `turn.complete()` keeps it before rethrowing, instead of dropping the streamed-so-far output. ([#4348](https://github.com/triggerdotdev/trigger.dev/pull/4348)) ## Server changes These changes affect the self-hosted Docker image and Trigger.dev Cloud: - Favorite any dashboard page to a new Favorites section in the side menu, and customize the sidebar by renaming favorites, hiding items, and reordering items and sections. ([#4375](https://github.com/triggerdotdev/trigger.dev/pull/4375)) - List API endpoints now clamp the page size to a maximum of 100. Requests asking for a larger page size return up to 100 items and keep paginating, rather than pulling an unbounded page. ([#4360](https://github.com/triggerdotdev/trigger.dev/pull/4360)) - Organizations without billing alerts now get default spend alert thresholds, so you're notified before usage grows unexpectedly. The billing limit page no longer pre-selects an option before you've set a limit and prompts you to configure one. Alert previews now update immediately after you change your billing limit. ([#4328](https://github.com/triggerdotdev/trigger.dev/pull/4328)) - When you create a Personal Access Token, the generated token now shows its first and last few characters instead of being fully hidden, so you can confirm you copied the right value. ([#4363](https://github.com/triggerdotdev/trigger.dev/pull/4363)) - Add metrics to the realtime backend that measure how often a single changed run is served to multiple subscriptions in one batch. ([#4341](https://github.com/triggerdotdev/trigger.dev/pull/4341)) - Realtime run subscriptions can now be configured to read run data straight from the primary database, so a run's latest state is never served from a lagging replica. Off by default; replica reads are unchanged unless you turn it on. ([#4378](https://github.com/triggerdotdev/trigger.dev/pull/4378)) - SSO and Directory Sync are no longer restricted to Enterprise plans — get in touch and we can turn them on for your organization whatever plan you're on. ([#4393](https://github.com/triggerdotdev/trigger.dev/pull/4393)) - Improved supervisor observability: it now reports metrics for its outbound requests, making failed calls to upstream services easier to monitor. ([#4350](https://github.com/triggerdotdev/trigger.dev/pull/4350)) - The runs list on a task's page now updates live — run statuses change and newly triggered runs appear without a manual refresh, matching the main Runs page. ([#4377](https://github.com/triggerdotdev/trigger.dev/pull/4377)) - Speed up the Batches list page for environments with a large number of batches, which could previously time out while loading. ([#4361](https://github.com/triggerdotdev/trigger.dev/pull/4361)) - Container startup no longer prints database and ClickHouse connection strings (with credentials) to the logs. ([#4346](https://github.com/triggerdotdev/trigger.dev/pull/4346)) - The tasks page no longer runs two queries whose results were never displayed, cutting wasted work on every page load and removing a source of hidden server errors ([#4380](https://github.com/triggerdotdev/trigger.dev/pull/4380))
Raw changeset output # Releases ## @trigger.dev/build@4.5.8 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.8` ## trigger.dev@4.5.8 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.8` - `@trigger.dev/build@4.5.8` - `@trigger.dev/schema-to-json@4.5.8` ## @trigger.dev/core@4.5.8 ### Patch Changes - Allow additional environment API keys to create scoped public access tokens through the Trigger.dev API. Use server-issued public access tokens for batch operations so environment-scoped API keys can read batch results. ([#4387](https://github.com/triggerdotdev/trigger.dev/pull/4387)) ## @trigger.dev/python@4.5.8 ### Patch Changes - Updated dependencies: - `@trigger.dev/sdk@4.5.8` - `@trigger.dev/core@4.5.8` - `@trigger.dev/build@4.5.8` ## @trigger.dev/react-hooks@4.5.8 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.8` ## @trigger.dev/redis-worker@4.5.8 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.8` ## @trigger.dev/rsc@4.5.8 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.8` ## @trigger.dev/schema-to-json@4.5.8 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.8` ## @trigger.dev/sdk@4.5.8 ### Patch Changes - Preserve the partial assistant message when a chat turn's model stream fails mid-response. `chat.agent` now passes the recovered partial to `onTurnComplete`, and `chat.createSession`'s `turn.complete()` keeps it before rethrowing, instead of dropping the streamed-so-far output. ([#4348](https://github.com/triggerdotdev/trigger.dev/pull/4348)) - Allow additional environment API keys to create scoped public access tokens through the Trigger.dev API. Use server-issued public access tokens for batch operations so environment-scoped API keys can read batch results. ([#4387](https://github.com/triggerdotdev/trigger.dev/pull/4387)) - Updated dependencies: - `@trigger.dev/core@4.5.8`
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- ...gent-preserve-partial-on-stream-failure.md | 5 ---- .changeset/friendly-keys-batch-tokens.md | 6 ----- .../batch-list-created-at-index.md | 6 ----- .../cap-list-endpoint-page-size.md | 6 ----- .server-changes/default-billing-alerts.md | 6 ----- ...entrypoint-no-log-db-connection-strings.md | 6 ----- ...avorite-pages-and-sidebar-customization.md | 6 ----- .../pat-token-reveal-last-chars.md | 6 ----- .../realtime-emission-fanout-metrics.md | 6 ----- .../realtime-run-reads-from-primary.md | 6 ----- .server-changes/sso-entitlement.md | 6 ----- .../supervisor-outbound-request-metrics.md | 6 ----- ...task-list-remove-unused-metrics-queries.md | 6 ----- .../task-page-runs-live-updating.md | 6 ----- hosting/k8s/helm/Chart.yaml | 4 ++-- packages/build/CHANGELOG.md | 7 ++++++ packages/build/package.json | 4 ++-- packages/cli-v3/CHANGELOG.md | 9 +++++++ packages/cli-v3/package.json | 8 +++---- packages/core/CHANGELOG.md | 6 +++++ packages/core/package.json | 2 +- packages/python/CHANGELOG.md | 9 +++++++ packages/python/package.json | 12 +++++----- packages/react-hooks/CHANGELOG.md | 7 ++++++ packages/react-hooks/package.json | 4 ++-- packages/redis-worker/CHANGELOG.md | 7 ++++++ packages/redis-worker/package.json | 4 ++-- packages/rsc/CHANGELOG.md | 7 ++++++ packages/rsc/package.json | 6 ++--- packages/schema-to-json/CHANGELOG.md | 7 ++++++ packages/schema-to-json/package.json | 2 +- packages/trigger-sdk/CHANGELOG.md | 9 +++++++ packages/trigger-sdk/package.json | 4 ++-- pnpm-lock.yaml | 24 +++++++++---------- 34 files changed, 105 insertions(+), 120 deletions(-) delete mode 100644 .changeset/chat-agent-preserve-partial-on-stream-failure.md delete mode 100644 .changeset/friendly-keys-batch-tokens.md delete mode 100644 .server-changes/batch-list-created-at-index.md delete mode 100644 .server-changes/cap-list-endpoint-page-size.md delete mode 100644 .server-changes/default-billing-alerts.md delete mode 100644 .server-changes/entrypoint-no-log-db-connection-strings.md delete mode 100644 .server-changes/favorite-pages-and-sidebar-customization.md delete mode 100644 .server-changes/pat-token-reveal-last-chars.md delete mode 100644 .server-changes/realtime-emission-fanout-metrics.md delete mode 100644 .server-changes/realtime-run-reads-from-primary.md delete mode 100644 .server-changes/sso-entitlement.md delete mode 100644 .server-changes/supervisor-outbound-request-metrics.md delete mode 100644 .server-changes/task-list-remove-unused-metrics-queries.md delete mode 100644 .server-changes/task-page-runs-live-updating.md diff --git a/.changeset/chat-agent-preserve-partial-on-stream-failure.md b/.changeset/chat-agent-preserve-partial-on-stream-failure.md deleted file mode 100644 index 3e7defddecd..00000000000 --- a/.changeset/chat-agent-preserve-partial-on-stream-failure.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@trigger.dev/sdk": patch ---- - -Preserve the partial assistant message when a chat turn's model stream fails mid-response. `chat.agent` now passes the recovered partial to `onTurnComplete`, and `chat.createSession`'s `turn.complete()` keeps it before rethrowing, instead of dropping the streamed-so-far output. diff --git a/.changeset/friendly-keys-batch-tokens.md b/.changeset/friendly-keys-batch-tokens.md deleted file mode 100644 index 505edacd334..00000000000 --- a/.changeset/friendly-keys-batch-tokens.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@trigger.dev/core": patch -"@trigger.dev/sdk": patch ---- - -Allow additional environment API keys to create scoped public access tokens through the Trigger.dev API. Use server-issued public access tokens for batch operations so environment-scoped API keys can read batch results. diff --git a/.server-changes/batch-list-created-at-index.md b/.server-changes/batch-list-created-at-index.md deleted file mode 100644 index b5a24386614..00000000000 --- a/.server-changes/batch-list-created-at-index.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -Speed up the Batches list page for environments with a large number of batches, which could previously time out while loading. diff --git a/.server-changes/cap-list-endpoint-page-size.md b/.server-changes/cap-list-endpoint-page-size.md deleted file mode 100644 index 2649f305787..00000000000 --- a/.server-changes/cap-list-endpoint-page-size.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: improvement ---- - -List API endpoints now clamp the page size to a maximum of 100. Requests asking for a larger page size return up to 100 items and keep paginating, rather than pulling an unbounded page. diff --git a/.server-changes/default-billing-alerts.md b/.server-changes/default-billing-alerts.md deleted file mode 100644 index e828c506050..00000000000 --- a/.server-changes/default-billing-alerts.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: improvement ---- - -Organizations without billing alerts now get default spend alert thresholds, so you're notified before usage grows unexpectedly. The billing limit page no longer pre-selects an option before you've set a limit and prompts you to configure one. Alert previews now update immediately after you change your billing limit. diff --git a/.server-changes/entrypoint-no-log-db-connection-strings.md b/.server-changes/entrypoint-no-log-db-connection-strings.md deleted file mode 100644 index 611408cf14c..00000000000 --- a/.server-changes/entrypoint-no-log-db-connection-strings.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -Container startup no longer prints database and ClickHouse connection strings (with credentials) to the logs. diff --git a/.server-changes/favorite-pages-and-sidebar-customization.md b/.server-changes/favorite-pages-and-sidebar-customization.md deleted file mode 100644 index d4468b4df51..00000000000 --- a/.server-changes/favorite-pages-and-sidebar-customization.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: feature ---- - -Favorite any dashboard page to a new Favorites section in the side menu, and customize the sidebar by renaming favorites, hiding items, and reordering items and sections. diff --git a/.server-changes/pat-token-reveal-last-chars.md b/.server-changes/pat-token-reveal-last-chars.md deleted file mode 100644 index 29797f73d34..00000000000 --- a/.server-changes/pat-token-reveal-last-chars.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: improvement ---- - -When you create a Personal Access Token, the generated token now shows its first and last few characters instead of being fully hidden, so you can confirm you copied the right value. diff --git a/.server-changes/realtime-emission-fanout-metrics.md b/.server-changes/realtime-emission-fanout-metrics.md deleted file mode 100644 index bd9ef83afbc..00000000000 --- a/.server-changes/realtime-emission-fanout-metrics.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: improvement ---- - -Add metrics to the realtime backend that measure how often a single changed run is served to multiple subscriptions in one batch. diff --git a/.server-changes/realtime-run-reads-from-primary.md b/.server-changes/realtime-run-reads-from-primary.md deleted file mode 100644 index c52ea44078b..00000000000 --- a/.server-changes/realtime-run-reads-from-primary.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: improvement ---- - -Realtime run subscriptions can now be configured to read run data straight from the primary database, so a run's latest state is never served from a lagging replica. Off by default; replica reads are unchanged unless you turn it on. diff --git a/.server-changes/sso-entitlement.md b/.server-changes/sso-entitlement.md deleted file mode 100644 index 4a039b6855e..00000000000 --- a/.server-changes/sso-entitlement.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: improvement ---- - -SSO and Directory Sync are no longer restricted to Enterprise plans — get in touch and we can turn them on for your organization whatever plan you're on. diff --git a/.server-changes/supervisor-outbound-request-metrics.md b/.server-changes/supervisor-outbound-request-metrics.md deleted file mode 100644 index c66a499b815..00000000000 --- a/.server-changes/supervisor-outbound-request-metrics.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: supervisor -type: improvement ---- - -Improved supervisor observability: it now reports metrics for its outbound requests, making failed calls to upstream services easier to monitor. diff --git a/.server-changes/task-list-remove-unused-metrics-queries.md b/.server-changes/task-list-remove-unused-metrics-queries.md deleted file mode 100644 index 23164a205ca..00000000000 --- a/.server-changes/task-list-remove-unused-metrics-queries.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -The tasks page no longer runs two queries whose results were never displayed, cutting wasted work on every page load and removing a source of hidden server errors diff --git a/.server-changes/task-page-runs-live-updating.md b/.server-changes/task-page-runs-live-updating.md deleted file mode 100644 index a082fdfb3d5..00000000000 --- a/.server-changes/task-page-runs-live-updating.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: improvement ---- - -The runs list on a task's page now updates live — run statuses change and newly triggered runs appear without a manual refresh, matching the main Runs page. diff --git a/hosting/k8s/helm/Chart.yaml b/hosting/k8s/helm/Chart.yaml index e40b51845c3..69538c1a239 100644 --- a/hosting/k8s/helm/Chart.yaml +++ b/hosting/k8s/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: trigger description: The official Trigger.dev Helm chart type: application -version: 4.5.7 -appVersion: v4.5.7 +version: 4.5.8 +appVersion: v4.5.8 home: https://trigger.dev sources: - https://github.com/triggerdotdev/trigger.dev diff --git a/packages/build/CHANGELOG.md b/packages/build/CHANGELOG.md index c8bbf39dd05..10f67297541 100644 --- a/packages/build/CHANGELOG.md +++ b/packages/build/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/build +## 4.5.8 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.8` + ## 4.5.7 ### Patch Changes diff --git a/packages/build/package.json b/packages/build/package.json index e598555da86..39c7e5cd703 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/build", - "version": "4.5.7", + "version": "4.5.8", "description": "trigger.dev build extensions", "license": "MIT", "publishConfig": { @@ -79,7 +79,7 @@ }, "dependencies": { "@prisma/config": "^6.10.0", - "@trigger.dev/core": "workspace:4.5.7", + "@trigger.dev/core": "workspace:4.5.8", "mlly": "^1.7.1", "pkg-types": "^1.1.3", "resolve": "^1.22.8", diff --git a/packages/cli-v3/CHANGELOG.md b/packages/cli-v3/CHANGELOG.md index 5840b31cdf6..c3821d48b18 100644 --- a/packages/cli-v3/CHANGELOG.md +++ b/packages/cli-v3/CHANGELOG.md @@ -1,5 +1,14 @@ # trigger.dev +## 4.5.8 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.8` + - `@trigger.dev/build@4.5.8` + - `@trigger.dev/schema-to-json@4.5.8` + ## 4.5.7 ### Patch Changes diff --git a/packages/cli-v3/package.json b/packages/cli-v3/package.json index c6d8d9719dc..9dba77e6af7 100644 --- a/packages/cli-v3/package.json +++ b/packages/cli-v3/package.json @@ -1,6 +1,6 @@ { "name": "trigger.dev", - "version": "4.5.7", + "version": "4.5.8", "description": "A Command-Line Interface for Trigger.dev projects", "type": "module", "license": "MIT", @@ -96,9 +96,9 @@ "@opentelemetry/sdk-trace-node": "2.7.1", "@opentelemetry/semantic-conventions": "1.41.1", "@s2-dev/streamstore": "^0.22.10", - "@trigger.dev/build": "workspace:4.5.7", - "@trigger.dev/core": "workspace:4.5.7", - "@trigger.dev/schema-to-json": "workspace:4.5.7", + "@trigger.dev/build": "workspace:4.5.8", + "@trigger.dev/core": "workspace:4.5.8", + "@trigger.dev/schema-to-json": "workspace:4.5.8", "ansi-escapes": "^7.0.0", "braces": "^3.0.3", "c12": "^1.11.1", diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index 6c2fb47dd01..91222935306 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -1,5 +1,11 @@ # internal-platform +## 4.5.8 + +### Patch Changes + +- Allow additional environment API keys to create scoped public access tokens through the Trigger.dev API. Use server-issued public access tokens for batch operations so environment-scoped API keys can read batch results. ([#4387](https://github.com/triggerdotdev/trigger.dev/pull/4387)) + ## 4.5.7 ### Patch Changes diff --git a/packages/core/package.json b/packages/core/package.json index f0c3adfcf2f..c99f83fe658 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/core", - "version": "4.5.7", + "version": "4.5.8", "description": "Core code used across the Trigger.dev SDK and platform", "license": "MIT", "publishConfig": { diff --git a/packages/python/CHANGELOG.md b/packages/python/CHANGELOG.md index bd9c54d71c7..ef31e38a05c 100644 --- a/packages/python/CHANGELOG.md +++ b/packages/python/CHANGELOG.md @@ -1,5 +1,14 @@ # @trigger.dev/python +## 4.5.8 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/sdk@4.5.8` + - `@trigger.dev/core@4.5.8` + - `@trigger.dev/build@4.5.8` + ## 4.5.7 ### Patch Changes diff --git a/packages/python/package.json b/packages/python/package.json index bebd4ade64f..3b1fbb67823 100644 --- a/packages/python/package.json +++ b/packages/python/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/python", - "version": "4.5.7", + "version": "4.5.8", "description": "Python runtime and build extension for Trigger.dev", "license": "MIT", "publishConfig": { @@ -45,7 +45,7 @@ "check-exports": "attw --pack ." }, "dependencies": { - "@trigger.dev/core": "workspace:4.5.7", + "@trigger.dev/core": "workspace:4.5.8", "tinyexec": "^0.3.2" }, "devDependencies": { @@ -56,12 +56,12 @@ "tsx": "4.17.0", "esbuild": "^0.23.0", "@arethetypeswrong/cli": "^0.18.5", - "@trigger.dev/build": "workspace:4.5.7", - "@trigger.dev/sdk": "workspace:4.5.7" + "@trigger.dev/build": "workspace:4.5.8", + "@trigger.dev/sdk": "workspace:4.5.8" }, "peerDependencies": { - "@trigger.dev/sdk": "workspace:^4.5.7", - "@trigger.dev/build": "workspace:^4.5.7" + "@trigger.dev/sdk": "workspace:^4.5.8", + "@trigger.dev/build": "workspace:^4.5.8" }, "engines": { "node": ">=18.20.0" diff --git a/packages/react-hooks/CHANGELOG.md b/packages/react-hooks/CHANGELOG.md index f422e8cb94f..78140e5a18b 100644 --- a/packages/react-hooks/CHANGELOG.md +++ b/packages/react-hooks/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/react-hooks +## 4.5.8 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.8` + ## 4.5.7 ### Patch Changes diff --git a/packages/react-hooks/package.json b/packages/react-hooks/package.json index b0b0ae668d9..48a5b67a6b1 100644 --- a/packages/react-hooks/package.json +++ b/packages/react-hooks/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/react-hooks", - "version": "4.5.7", + "version": "4.5.8", "description": "trigger.dev react hooks", "license": "MIT", "publishConfig": { @@ -37,7 +37,7 @@ "check-exports": "attw --pack ." }, "dependencies": { - "@trigger.dev/core": "workspace:^4.5.7", + "@trigger.dev/core": "workspace:^4.5.8", "swr": "^2.2.5" }, "devDependencies": { diff --git a/packages/redis-worker/CHANGELOG.md b/packages/redis-worker/CHANGELOG.md index 5ddc4f32d44..1fbed6edc3e 100644 --- a/packages/redis-worker/CHANGELOG.md +++ b/packages/redis-worker/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/redis-worker +## 4.5.8 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.8` + ## 4.5.7 ### Patch Changes diff --git a/packages/redis-worker/package.json b/packages/redis-worker/package.json index edd27f40e41..7d2ff4dceea 100644 --- a/packages/redis-worker/package.json +++ b/packages/redis-worker/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/redis-worker", - "version": "4.5.7", + "version": "4.5.8", "description": "Redis worker for trigger.dev", "license": "MIT", "publishConfig": { @@ -23,7 +23,7 @@ "test": "vitest --sequence.concurrent=false --no-file-parallelism" }, "dependencies": { - "@trigger.dev/core": "workspace:4.5.7", + "@trigger.dev/core": "workspace:4.5.8", "lodash.omit": "^4.5.0", "nanoid": "^5.0.7", "p-limit": "^6.2.0", diff --git a/packages/rsc/CHANGELOG.md b/packages/rsc/CHANGELOG.md index 89b0364191d..f30b7980f35 100644 --- a/packages/rsc/CHANGELOG.md +++ b/packages/rsc/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/rsc +## 4.5.8 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.8` + ## 4.5.7 ### Patch Changes diff --git a/packages/rsc/package.json b/packages/rsc/package.json index 46b17e36ba9..b216d219d8c 100644 --- a/packages/rsc/package.json +++ b/packages/rsc/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/rsc", - "version": "4.5.7", + "version": "4.5.8", "description": "trigger.dev rsc", "license": "MIT", "publishConfig": { @@ -37,14 +37,14 @@ "check-exports": "attw --pack ." }, "dependencies": { - "@trigger.dev/core": "workspace:^4.5.7", + "@trigger.dev/core": "workspace:^4.5.8", "mlly": "^1.7.1", "react": "19.0.0-rc.1", "react-dom": "19.0.0-rc.1" }, "devDependencies": { "@arethetypeswrong/cli": "^0.18.5", - "@trigger.dev/build": "workspace:^4.5.7", + "@trigger.dev/build": "workspace:^4.5.8", "@types/node": "^24.13.3", "@types/react": "*", "@types/react-dom": "*", diff --git a/packages/schema-to-json/CHANGELOG.md b/packages/schema-to-json/CHANGELOG.md index 50c400b068b..6f5a7cc5bae 100644 --- a/packages/schema-to-json/CHANGELOG.md +++ b/packages/schema-to-json/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/schema-to-json +## 4.5.8 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.8` + ## 4.5.7 ### Patch Changes diff --git a/packages/schema-to-json/package.json b/packages/schema-to-json/package.json index 03e9c936210..66aa20c358f 100644 --- a/packages/schema-to-json/package.json +++ b/packages/schema-to-json/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/schema-to-json", - "version": "4.5.7", + "version": "4.5.8", "description": "Convert various schema validation libraries to JSON Schema", "license": "MIT", "publishConfig": { diff --git a/packages/trigger-sdk/CHANGELOG.md b/packages/trigger-sdk/CHANGELOG.md index ecb8fc3bacf..fa4b2ddb99f 100644 --- a/packages/trigger-sdk/CHANGELOG.md +++ b/packages/trigger-sdk/CHANGELOG.md @@ -1,5 +1,14 @@ # @trigger.dev/sdk +## 4.5.8 + +### Patch Changes + +- Preserve the partial assistant message when a chat turn's model stream fails mid-response. `chat.agent` now passes the recovered partial to `onTurnComplete`, and `chat.createSession`'s `turn.complete()` keeps it before rethrowing, instead of dropping the streamed-so-far output. ([#4348](https://github.com/triggerdotdev/trigger.dev/pull/4348)) +- Allow additional environment API keys to create scoped public access tokens through the Trigger.dev API. Use server-issued public access tokens for batch operations so environment-scoped API keys can read batch results. ([#4387](https://github.com/triggerdotdev/trigger.dev/pull/4387)) +- Updated dependencies: + - `@trigger.dev/core@4.5.8` + ## 4.5.7 ### Patch Changes diff --git a/packages/trigger-sdk/package.json b/packages/trigger-sdk/package.json index 50141b73cd0..41b5591c8ad 100644 --- a/packages/trigger-sdk/package.json +++ b/packages/trigger-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/sdk", - "version": "4.5.7", + "version": "4.5.8", "description": "trigger.dev Node.JS SDK", "license": "MIT", "publishConfig": { @@ -77,7 +77,7 @@ "dependencies": { "@opentelemetry/api": "1.9.1", "@opentelemetry/semantic-conventions": "1.41.1", - "@trigger.dev/core": "workspace:4.5.7", + "@trigger.dev/core": "workspace:4.5.8", "chalk": "^5.2.0", "cronstrue": "^2.21.0", "debug": "^4.3.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 204cae651c0..2849a27dd58 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1419,7 +1419,7 @@ importers: specifier: ^6.10.0 version: 6.19.0(magicast@0.3.5) '@trigger.dev/core': - specifier: workspace:4.5.7 + specifier: workspace:4.5.8 version: link:../core mlly: specifier: ^1.7.1 @@ -1495,13 +1495,13 @@ importers: specifier: ^0.22.10 version: 0.22.10(supports-color@10.0.0) '@trigger.dev/build': - specifier: workspace:4.5.7 + specifier: workspace:4.5.8 version: link:../build '@trigger.dev/core': - specifier: workspace:4.5.7 + specifier: workspace:4.5.8 version: link:../core '@trigger.dev/schema-to-json': - specifier: workspace:4.5.7 + specifier: workspace:4.5.8 version: link:../schema-to-json ansi-escapes: specifier: ^7.0.0 @@ -1888,7 +1888,7 @@ importers: packages/python: dependencies: '@trigger.dev/core': - specifier: workspace:4.5.7 + specifier: workspace:4.5.8 version: link:../core tinyexec: specifier: ^0.3.2 @@ -1898,10 +1898,10 @@ importers: specifier: ^0.18.5 version: 0.18.5 '@trigger.dev/build': - specifier: workspace:4.5.7 + specifier: workspace:4.5.8 version: link:../build '@trigger.dev/sdk': - specifier: workspace:4.5.7 + specifier: workspace:4.5.8 version: link:../trigger-sdk '@types/node': specifier: 24.13.3 @@ -1925,7 +1925,7 @@ importers: packages/react-hooks: dependencies: '@trigger.dev/core': - specifier: workspace:^4.5.7 + specifier: workspace:^4.5.8 version: link:../core react: specifier: 18.3.1 @@ -1959,7 +1959,7 @@ importers: packages/redis-worker: dependencies: '@trigger.dev/core': - specifier: workspace:4.5.7 + specifier: workspace:4.5.8 version: link:../core cron-parser: specifier: ^4.9.0 @@ -2008,7 +2008,7 @@ importers: packages/rsc: dependencies: '@trigger.dev/core': - specifier: workspace:^4.5.7 + specifier: workspace:^4.5.8 version: link:../core mlly: specifier: ^1.7.1 @@ -2024,7 +2024,7 @@ importers: specifier: ^0.18.5 version: 0.18.5 '@trigger.dev/build': - specifier: workspace:^4.5.7 + specifier: workspace:^4.5.8 version: link:../build '@types/node': specifier: 24.13.3 @@ -2103,7 +2103,7 @@ importers: specifier: 1.41.1 version: 1.41.1 '@trigger.dev/core': - specifier: workspace:4.5.7 + specifier: workspace:4.5.8 version: link:../core chalk: specifier: ^5.2.0 From fc576436e24c0fa4d29fffa2a50bc627b3626711 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Mon, 27 Jul 2026 16:59:43 +0100 Subject: [PATCH 20/67] perf(run-ops-database): index BatchTaskRun for the batches list on the dedicated schema (#4396) ## Summary The batches list page orders by `(createdAt DESC, id DESC)`, which is why [#4361](https://github.com/triggerdotdev/trigger.dev/pull/4361) added a matching index on `BatchTaskRun`. That index only landed in `@trigger.dev/database`. The dedicated run-ops database has its own migration history, so it never received the index. `BatchListPresenter` reads both databases and merges, so for environments whose batches live in the dedicated database the page kept falling back to a scan and in-memory sort, which is the exact behaviour #4361 set out to fix. ## Fix Adds the index to the run-ops schema with its own migration. `CREATE INDEX CONCURRENTLY IF NOT EXISTS`, so it is a no-op where the index already exists and still records its ledger row. The second half is the interesting part. Because the two packages own separate migration histories, a run-graph schema change has to be authored twice, and nothing made the miss visible: the run-ops status check truthfully reports "up to date" against its own history, so the apply step just skips. `schemaParity.test.ts` compares the physical shape of every model the run-ops schema declares against its counterpart in `@trigger.dev/database`: scalar fields with their attributes, plus `@@index`, `@@unique`, `@@id` and `@@map`. Relation navigation fields are excluded, since the run-ops schema deliberately drops relations that would cross a database boundary while keeping the scalar FK column. A field counts as a relation when its type resolves to a model name, which keeps enum-typed columns in scope. Two models are listed as run-ops-only: `CompletedWaitpoint` and `WaitpointRunConnection`, both explicit FK-free replacements for a control-plane implicit many-to-many, since an implicit m2m carries a foreign key that cannot resolve across databases. The test also asserts that exception list is exhaustive, so a new unpaired model fails rather than being silently skipped. Confirmed the guard actually fails: reverting the index turns `BatchTaskRun` red with the missing `@@index` named in the diff. --- .../migration.sql | 1 + .../run-ops-database/prisma/schema.prisma | 1 + .../run-ops-database/src/schemaParity.test.ts | 135 ++++++++++++++++++ 3 files changed, 137 insertions(+) create mode 100644 internal-packages/run-ops-database/prisma/migrations/20260727120000_add_batch_task_run_created_at_dashboard_index/migration.sql create mode 100644 internal-packages/run-ops-database/src/schemaParity.test.ts diff --git a/internal-packages/run-ops-database/prisma/migrations/20260727120000_add_batch_task_run_created_at_dashboard_index/migration.sql b/internal-packages/run-ops-database/prisma/migrations/20260727120000_add_batch_task_run_created_at_dashboard_index/migration.sql new file mode 100644 index 00000000000..0d6776d9a78 --- /dev/null +++ b/internal-packages/run-ops-database/prisma/migrations/20260727120000_add_batch_task_run_created_at_dashboard_index/migration.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS "BatchTaskRun_runtimeEnvironmentId_createdAt_id_idx" ON "public"."BatchTaskRun"("runtimeEnvironmentId", "createdAt" DESC, "id" DESC); diff --git a/internal-packages/run-ops-database/prisma/schema.prisma b/internal-packages/run-ops-database/prisma/schema.prisma index 8e49fc854e6..4750efa392c 100644 --- a/internal-packages/run-ops-database/prisma/schema.prisma +++ b/internal-packages/run-ops-database/prisma/schema.prisma @@ -827,6 +827,7 @@ model BatchTaskRun { @@index([dependentTaskAttemptId]) // This is for the batch list dashboard page @@index([runtimeEnvironmentId, id(sort: Desc)]) + @@index([runtimeEnvironmentId, createdAt(sort: Desc), id(sort: Desc)]) } enum BatchTaskRunStatus { diff --git a/internal-packages/run-ops-database/src/schemaParity.test.ts b/internal-packages/run-ops-database/src/schemaParity.test.ts new file mode 100644 index 00000000000..763fba73e0b --- /dev/null +++ b/internal-packages/run-ops-database/src/schemaParity.test.ts @@ -0,0 +1,135 @@ +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +/** + * Guards the double-authoring rule for run-graph schema changes. + * + * The run-ops database is on its own migration history, so a change to a run-graph model has to be + * written twice: once in `@trigger.dev/database` (which targets the control-plane and legacy RDS + * databases) and once here. Nothing about the two histories forces that, and when it is missed the + * run-ops status job truthfully reports "up to date" against its own history — so the change simply + * never lands, silently. + * + * This compares the PHYSICAL shape of every model the run-ops schema declares against its + * control-plane counterpart: scalar fields with their attributes, plus `@@index` / `@@unique` / + * `@@id` / `@@map`. Relation navigation fields are excluded, because the run-ops schema deliberately + * drops relations that would cross into the control-plane database while keeping the scalar FK + * column. + */ + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const RUN_OPS_SCHEMA = resolve(packageRoot, "prisma/schema.prisma"); +const CONTROL_PLANE_SCHEMA = resolve(packageRoot, "../database/prisma/schema.prisma"); + +/** + * Models that exist only in the run-ops schema, with no control-plane counterpart to compare + * against. Both replace a control-plane implicit many-to-many relation with an explicit, FK-free + * join model, because an implicit m2m carries a foreign key that cannot resolve across databases. + */ +const RUN_OPS_ONLY_MODELS = new Set(["CompletedWaitpoint", "WaitpointRunConnection"]); + +type Schema = { + models: Map; + modelNames: Set; +}; + +function parseSchema(path: string): Schema { + const lines = readFileSync(path, "utf8").split("\n"); + const models = new Map(); + let current: string | null = null; + let body: string[] = []; + + for (const line of lines) { + const open = line.match(/^model\s+(\w+)\s*\{/); + if (open) { + current = open[1]!; + body = []; + continue; + } + if (current !== null && /^\}/.test(line)) { + models.set(current, body); + current = null; + continue; + } + if (current !== null) { + const trimmed = line.trim(); + if (trimmed.length > 0 && !trimmed.startsWith("//")) { + body.push(trimmed); + } + } + } + + return { models, modelNames: new Set(models.keys()) }; +} + +/** + * Keeps the lines that describe physical database structure and drops the ones that describe Prisma + * relations. A field is a relation navigation field when its type resolves to a model, so scalar FK + * columns (`String`) and enum-typed columns are both retained. + */ +function physicalLines(body: string[], modelNames: Set): Set { + const kept = new Set(); + + for (const line of body) { + if (line.startsWith("@@")) { + if (/^@@(index|unique|id|map)\b/.test(line)) { + kept.add(line.replace(/\s+/g, " ")); + } + continue; + } + + const field = line.match(/^(\w+)\s+(\w+)(\[\]|\?)?/); + if (!field) continue; + + const [, , type] = field; + if (modelNames.has(type!)) continue; + if (line.includes("@relation(")) continue; + + kept.add(line.replace(/\s+/g, " ")); + } + + return kept; +} + +describe("run-ops schema parity with the control-plane schema", () => { + const runOps = parseSchema(RUN_OPS_SCHEMA); + const controlPlane = parseSchema(CONTROL_PLANE_SCHEMA); + + const sharedModels = [...runOps.models.keys()] + .filter((name) => !RUN_OPS_ONLY_MODELS.has(name)) + .sort(); + + it("finds models to compare", () => { + expect(sharedModels.length).toBeGreaterThan(10); + }); + + it("declares every run-ops-only model in the exception list, and no others", () => { + const missingFromControlPlane = [...runOps.models.keys()] + .filter((name) => !controlPlane.models.has(name)) + .sort(); + + expect(missingFromControlPlane).toEqual([...RUN_OPS_ONLY_MODELS].sort()); + }); + + it.each(sharedModels)("%s has the same physical shape in both schemas", (modelName) => { + const runOpsBody = runOps.models.get(modelName); + const controlPlaneBody = controlPlane.models.get(modelName); + + expect(runOpsBody, `${modelName} missing from the run-ops schema`).toBeDefined(); + expect(controlPlaneBody, `${modelName} missing from the control-plane schema`).toBeDefined(); + + const runOpsShape = physicalLines(runOpsBody!, runOps.modelNames); + const controlPlaneShape = physicalLines(controlPlaneBody!, controlPlane.modelNames); + + const onlyInControlPlane = [...controlPlaneShape].filter((l) => !runOpsShape.has(l)).sort(); + const onlyInRunOps = [...runOpsShape].filter((l) => !controlPlaneShape.has(l)).sort(); + + expect( + { onlyInControlPlane, onlyInRunOps }, + `${modelName} differs physically between the two schemas. A run-graph change was written to ` + + `one schema and not the other; add the missing side plus a migration in that package.` + ).toEqual({ onlyInControlPlane: [], onlyInRunOps: [] }); + }); +}); From 3ed48516dff5d27c85a2a689996e8707481a9783 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 27 Jul 2026 17:22:46 +0100 Subject: [PATCH 21/67] ci: let the claude bot trigger the PR audit workflows (#4392) ## Summary PRs opened by the claude GitHub app fail both the agent instructions audit and the REVIEW.md drift audit before Claude gets a chance to run. `claude-code-action` refuses any actor whose account type is not `User` unless the actor is listed in `allowed_bots`: ``` Workflow initiated by non-human actor: claude (type: Bot). Add bot to allowed_bots list or use '*' to allow all bots. ``` So those PRs land with two permanently red checks and no audit coverage at all. Both workflows already allowlist Devin; this adds the claude app alongside it. ## Why this does not open the workflows up to outside contributors `allowed_bots` is only consulted for non-`User` actors. Humans, contributor or maintainer, take the separate write-permission path and are unaffected by what is in the list. Beyond that, both jobs are guarded by `github.event.pull_request.head.repo.full_name == github.repository`, so a fork PR skips the job entirely, and they trigger on `pull_request` rather than `pull_request_target`, so a fork-triggered run would get no API key and a read-only token anyway. The bot is named explicitly instead of using `"*"`, which would let every bot trigger these audits, dependabot's PR stream included. --- .github/workflows/check-review-md.yml | 2 +- .github/workflows/claude-md-audit.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-review-md.yml b/.github/workflows/check-review-md.yml index 06f0d2d82c4..79b9d7eacd5 100644 --- a/.github/workflows/check-review-md.yml +++ b/.github/workflows/check-review-md.yml @@ -38,7 +38,7 @@ jobs: with: anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} use_sticky_comment: true - allowed_bots: "devin-ai-integration[bot]" + allowed_bots: "devin-ai-integration[bot],claude[bot]" claude_args: | --max-turns 30 diff --git a/.github/workflows/claude-md-audit.yml b/.github/workflows/claude-md-audit.yml index aa62444c9df..d1b1bbdb45a 100644 --- a/.github/workflows/claude-md-audit.yml +++ b/.github/workflows/claude-md-audit.yml @@ -40,7 +40,7 @@ jobs: with: anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} use_sticky_comment: true - allowed_bots: "devin-ai-integration[bot]" + allowed_bots: "devin-ai-integration[bot],claude[bot]" claude_args: | --max-turns 25 From ec562c0e68c1ef3f82227b865b2edc8f900695db Mon Sep 17 00:00:00 2001 From: "claude[bot]" <209825114+claude[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:00:48 +0100 Subject: [PATCH 22/67] fix(webapp): remove unused Electric sync trace routes (#4400) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _Requested by **Eric Allam** · [Slack thread](https://triggerdotdev.slack.com/archives/C0AU83M3136/p1785222101937829?thread_ts=1785207509.304669&cid=C0AU83M3136)_ Removes two dead Remix routes and the helpers only they used. `app/routes/sync.traces.runs.$traceId.ts` (`/sync/traces/runs/:traceId`) and `app/routes/sync.traces.$traceId.ts` (`/sync/traces/:traceId`) were added with the original ElectricSQL run page and lost their only consumers when the dashboard hooks that called them were deleted. Nothing in the repo references either route today. Also removed, because the deleted routes were their only callers: - `OtelTraceIdSchema`, `RESERVED_ELECTRIC_SHAPE_PARAMS`, `TraceScope`, `buildElectricTraceWhereClause` from `app/v3/electricShape.server.ts` (the file stays — `UNSAFE_REALTIME_TAG_CHARS` / `sanitizeRealtimeTagForSql` / `sanitizeRealtimeTagsForSql` are still used by `realtime.v1.runs.ts` and `realtimeClient.server.ts`) - the loader-specific cases in `apps/webapp/test/spanTraceRoutes.replicaLag.test.ts` and `internal-packages/run-store/src/runOpsStore.routesSpanTraceReadView.replicaLag.test.ts` `app/utils/longPollingFetch.ts` is untouched — `realtimeClient.server.ts` still uses it. `runOpsStore.ts` / `PostgresRunStore.ts` are untouched too; the unrouted-lookup mechanism there is generic and stays. As a plain code fact: the run lookup these loaders performed keyed on `TaskRun.traceId` alone, which is not an index-backed query shape. That is noted only as context for why the code is not worth keeping around unused. ### Judgement call worth a maintainer's opinion The request was specifically about `/sync/traces/runs/:traceId`, the route that looks up a run by `traceId`. This PR **also** deletes its sibling `/sync/traces/:traceId`. The reasoning: - both routes came in with the same ElectricSQL run-page work - both lost their only consumers in the same later commit - neither has any caller anywhere in the repo - they share the same helper module, so keeping one means keeping the helpers half-used If you would rather keep the sibling, reverting just that one file deletion is easy and does not affect the rest of this PR — say the word and I will restore it along with the helpers it needs. ## ✅ Checklist - [x] I have followed every step in the [contributing guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md) - [x] The PR title follows the convention. - [x] I ran and tested the code works --- ## Testing Verification run locally from the repo root: | Command | Result | | --- | --- | | `pnpm run format` | clean, no changes produced | | `pnpm run lint:fix` | clean | | `pnpm run lint` | pass (exit 0, no findings) | | `pnpm run typecheck --filter webapp` | pass | | `pnpm run typecheck --filter @internal/run-store` | pass | A ripgrep sweep for `sync.traces`, `sync/traces`, `syncTraceRunsLoader`, `buildElectricTraceWhereClause`, `OtelTraceIdSchema` and `RESERVED_ELECTRIC_SHAPE_PARAMS` (excluding `node_modules`) returns zero hits. **Not fully verified:** both edited test files are testcontainers suites and need a Docker runtime, which was not available in my environment. I confirmed each file *collects* correctly with exactly the three intended remaining tests and no import errors — notably, dropping the `session.server` / `controlPlaneResolver.server` / `longPollingFetch` / `env.server` mocks does not break module loading for the surviving loaders. The assertions themselves then failed only on `Could not find a working container runtime strategy`. CI should be the real signal here. Per `apps/webapp/CLAUDE.md`, `pnpm run build --filter webapp` was deliberately not run. --- ## Changelog Removed two unused sync routes left over from the original ElectricSQL run page, along with the helpers and tests that existed only to serve them. No behaviour change — neither route had any caller. --- ## Screenshots _n/a — no user-visible surface changes._ 💯 --------- Co-authored-by: Claude --- .../remove-unused-trace-sync-route.md | 6 + .../webapp/app/routes/sync.traces.$traceId.ts | 95 -------------- .../app/routes/sync.traces.runs.$traceId.ts | 117 ----------------- apps/webapp/app/v3/electricShape.server.ts | 44 ------- .../test/spanTraceRoutes.replicaLag.test.ts | 120 +----------------- ...routesSpanTraceReadView.replicaLag.test.ts | 54 +------- 6 files changed, 14 insertions(+), 422 deletions(-) create mode 100644 .server-changes/remove-unused-trace-sync-route.md delete mode 100644 apps/webapp/app/routes/sync.traces.$traceId.ts delete mode 100644 apps/webapp/app/routes/sync.traces.runs.$traceId.ts diff --git a/.server-changes/remove-unused-trace-sync-route.md b/.server-changes/remove-unused-trace-sync-route.md new file mode 100644 index 00000000000..b2e857405a0 --- /dev/null +++ b/.server-changes/remove-unused-trace-sync-route.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Removed an unused internal dashboard sync route that is no longer part of any product surface. diff --git a/apps/webapp/app/routes/sync.traces.$traceId.ts b/apps/webapp/app/routes/sync.traces.$traceId.ts deleted file mode 100644 index d7695014b56..00000000000 --- a/apps/webapp/app/routes/sync.traces.$traceId.ts +++ /dev/null @@ -1,95 +0,0 @@ -import type { LoaderFunctionArgs } from "@remix-run/node"; -import { z } from "zod"; -import { $replica } from "~/db.server"; -import { env } from "~/env.server"; -import { logger } from "~/services/logger.server"; -import { getUserId } from "~/services/session.server"; -import { longPollingFetch } from "~/utils/longPollingFetch"; -import { - OtelTraceIdSchema, - RESERVED_ELECTRIC_SHAPE_PARAMS, - buildElectricTraceWhereClause, -} from "~/v3/electricShape.server"; - -const Params = z.object({ - traceId: OtelTraceIdSchema, -}); - -export async function loader({ params, request }: LoaderFunctionArgs) { - try { - const userId = await getUserId(request); - - const parsedParams = Params.safeParse(params); - if (!parsedParams.success) { - // Treat a malformed traceId as not-found rather than 400 to avoid - // signalling the validator. - return new Response("Not found", { status: 404 }); - } - const { traceId } = parsedParams.data; - - logger.log(`/sync/traces/${traceId}`, { userId }); - - if (!userId) { - return new Response("No user found in cookie", { status: 401 }); - } - - const trace = await $replica.taskEvent.findFirst({ - select: { - organizationId: true, - }, - where: { - traceId, - }, - }); - - if (!trace) { - return new Response("No trace found", { status: 404 }); - } - - const member = await $replica.orgMember.findFirst({ - where: { - organizationId: trace.organizationId, - userId, - }, - }); - - if (!member) { - return new Response("Not a member of this org", { status: 401 }); - } - - const url = new URL(request.url); - const originUrl = new URL(`${env.ELECTRIC_ORIGIN}/v1/shape/public."TaskEvent"`); - // Strip params we set ourselves so the caller can't override them. - url.searchParams.forEach((value, key) => { - if (RESERVED_ELECTRIC_SHAPE_PARAMS.has(key)) return; - originUrl.searchParams.set(key, value); - }); - - originUrl.searchParams.set( - "where", - buildElectricTraceWhereClause({ - traceId, - scope: { column: "organizationId", id: trace.organizationId }, - }) - ); - - const finalUrl = originUrl.toString(); - - logger.log("Fetching trace data", { url: finalUrl }); - - return longPollingFetch(finalUrl); - } catch (error) { - if (error instanceof Response) { - // Error responses from longPollingFetch - return error; - } else if (error instanceof TypeError) { - // Unexpected errors - logger.error("Unexpected error in loader:", { error: error.message }); - return new Response("An unexpected error occurred", { status: 500 }); - } else { - // Unknown errors - logger.error("Unknown error occurred in loader, not Error", { error: JSON.stringify(error) }); - return new Response("An unknown error occurred", { status: 500 }); - } - } -} diff --git a/apps/webapp/app/routes/sync.traces.runs.$traceId.ts b/apps/webapp/app/routes/sync.traces.runs.$traceId.ts deleted file mode 100644 index 24870cf2f49..00000000000 --- a/apps/webapp/app/routes/sync.traces.runs.$traceId.ts +++ /dev/null @@ -1,117 +0,0 @@ -import type { LoaderFunctionArgs } from "@remix-run/node"; -import { z } from "zod"; -import { $replica } from "~/db.server"; -import { env } from "~/env.server"; -import { logger } from "~/services/logger.server"; -import { getUserId } from "~/services/session.server"; -import { longPollingFetch } from "~/utils/longPollingFetch"; -import { - OtelTraceIdSchema, - RESERVED_ELECTRIC_SHAPE_PARAMS, - buildElectricTraceWhereClause, -} from "~/v3/electricShape.server"; -import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; -import { runStore } from "~/v3/runStore.server"; - -const Params = z.object({ - traceId: OtelTraceIdSchema, -}); - -export async function loader({ params, request }: LoaderFunctionArgs) { - try { - const userId = await getUserId(request); - - const parsedParams = Params.safeParse(params); - if (!parsedParams.success) { - return new Response("Not found", { status: 404 }); - } - const { traceId } = parsedParams.data; - - logger.log(`/sync/runs/${traceId}`, { userId }); - - if (!userId) { - return new Response("No user found in cookie", { status: 401 }); - } - - let run = await runStore.findRun( - { - traceId, - }, - { - select: { - projectId: true, - runtimeEnvironmentId: true, - }, - }, - $replica - ); - - if (!run) { - // Read-your-writes: a just-created run may not have replicated yet. Re-read the owning - // primary before 404ing so a live run's realtime trace feed isn't spuriously not-found. - run = await runStore.findRunOnPrimary( - { traceId }, - { select: { projectId: true, runtimeEnvironmentId: true } } - ); - } - - if (!run) { - return new Response("No run found", { status: 404 }); - } - - const resolvedEnv = await controlPlaneResolver.resolveEnv(run.runtimeEnvironmentId); - - if (!resolvedEnv) { - return new Response("No run found", { status: 404 }); - } - - const member = await $replica.orgMember.findFirst({ - where: { - organizationId: resolvedEnv.organizationId, - userId, - }, - }); - - if (!member) { - return new Response("Not a member of this org", { status: 401 }); - } - - const url = new URL(request.url); - const originUrl = new URL(`${env.ELECTRIC_ORIGIN}/v1/shape/public."TaskRun"`); - // Strip params we set ourselves so the caller can't override them. - url.searchParams.forEach((value, key) => { - if (RESERVED_ELECTRIC_SHAPE_PARAMS.has(key)) return; - originUrl.searchParams.set(key, value); - }); - - originUrl.searchParams.set( - "where", - // Scope by non-null projectId, not the nullable organizationId (legacy - // rows would vanish). Tenant-safe: membership was verified against this - // project's org and a trace's runs all live in one project. - buildElectricTraceWhereClause({ - traceId, - scope: { column: "projectId", id: run.projectId }, - }) - ); - - const finalUrl = originUrl.toString(); - - logger.log("Fetching trace runs data", { url: finalUrl }); - - return longPollingFetch(finalUrl); - } catch (error) { - if (error instanceof Response) { - // Error responses from longPollingFetch - return error; - } else if (error instanceof TypeError) { - // Unexpected errors - logger.error("Unexpected error in loader:", { error: error.message }); - return new Response("An unexpected error occurred", { status: 500 }); - } else { - // Unknown errors - logger.error("Unknown error occurred in loader, not Error", { error: JSON.stringify(error) }); - return new Response("An unknown error occurred", { status: 500 }); - } - } -} diff --git a/apps/webapp/app/v3/electricShape.server.ts b/apps/webapp/app/v3/electricShape.server.ts index c430cc0273c..65d52032afb 100644 --- a/apps/webapp/app/v3/electricShape.server.ts +++ b/apps/webapp/app/v3/electricShape.server.ts @@ -1,47 +1,3 @@ -import { z } from "zod"; - -/** - * OTel trace IDs are 32 lowercase hex chars. The traceparent parser only - * checks the dash-delimited format, so crafted ids can be persisted and later - * interpolated into shape `where` clauses. Validate here to close the SQLi vector. - */ -export const OtelTraceIdSchema = z - .string() - .regex(/^[0-9a-f]{32}$/, "traceId must be 32 lowercase hex characters"); - -/** Params the sync routes set themselves; stripped from incoming requests. */ -export const RESERVED_ELECTRIC_SHAPE_PARAMS = new Set(["where", "table", "columns"]); - -const CUID_LIKE = /^[a-z][a-z0-9_]*$/i; - -/** - * Tenant column a trace shape is scoped by. TaskEvent scopes by non-null - * organizationId; TaskRun scopes by non-null projectId (its organizationId is - * nullable). The column is from this fixed union, never user input, so it's - * safe to interpolate. - */ -export type TraceScope = - | { column: "organizationId"; id: string } - | { column: "projectId"; id: string }; - -/** - * Build the Electric Shape `where` clause for the trace sync routes. Both ids - * are re-validated as defense-in-depth so a missed call site can't bypass scope. - */ -export function buildElectricTraceWhereClause(args: { - traceId: string; - scope: TraceScope; -}): string { - const { traceId, scope } = args; - if (!OtelTraceIdSchema.safeParse(traceId).success) { - throw new Error("buildElectricTraceWhereClause: unsafe traceId"); - } - if (!CUID_LIKE.test(scope.id)) { - throw new Error("buildElectricTraceWhereClause: unsafe scope id"); - } - return `"traceId"='${traceId}' AND "${scope.column}"='${scope.id}'`; -} - /** * Characters rejected in realtime tag values — the single source of truth * shared by the apiBuilder Zod refine (`realtime.v1.runs.ts`) and the runtime diff --git a/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts b/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts index 6b8dac27932..3c81ab3c5e4 100644 --- a/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts +++ b/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts @@ -2,12 +2,11 @@ // route caller (never a reimplemented read) against a real split Postgres with the owning LEGACY replica // FROZEN via laggingReplica. The routes pass a branded $replica, so reads stay on the owning replica. // Only orthogonal webapp singletons are mocked (bearer auth, mollifier buffer, ClickHouse repository, -// formatters, session cookie, control-plane resolver, longPollingFetch). +// formatters). // // Properties per read: the spans/trace findResource reads emit a RETRYABLE 404 (x-should-retry:true) on a // replica+buffer miss and return 200 once the replica catches up (proven with a caught-up store); the -// triggeredRuns list simply omits a just-triggered child under lag (200, eventually consistent); the sync -// trace-runs loader recovers a live run via a primary re-read on a replica miss and still 404s a truly-absent run. +// triggeredRuns list simply omits a just-triggered child under lag (200, eventually consistent). import { describe, expect, vi } from "vitest"; import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; @@ -20,20 +19,15 @@ vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); // ---- Hoisted holders wired into the mocked module singletons before each loader call. ------------- // The branded `$replica` marker uses the global-registry symbol the run-store brands replicas with -// (readReplicaClient.ts) so the routing store keeps the read on the owning REPLICA. It also carries a -// live `orgMember` delegate (pointed at the real prisma14) because the sync route reads -// `$replica.orgMember.findFirst` directly — orthogonal to the run read under test. +// (readReplicaClient.ts) so the routing store keeps the read on the owning REPLICA. const { holder } = vi.hoisted(() => { const REPLICA_BRAND = Symbol.for("trigger.dev/run-store/read-replica"); return { holder: { REPLICA_BRAND, store: undefined as unknown, - replicaMarker: undefined as unknown, environment: undefined as unknown, bufferResult: null as unknown, - userId: undefined as unknown, - resolvedEnv: undefined as unknown, span: undefined as unknown, traceSummary: undefined as unknown, }, @@ -57,7 +51,7 @@ vi.mock("~/v3/runStore.server", () => ({ ), })); -// `$replica` brand marker (routes it to the owning replica) + a live orgMember delegate for the sync route. +// `$replica` brand marker: routes the read to the owning replica. vi.mock("~/db.server", () => ({ prisma: {}, $replica: new Proxy( @@ -65,8 +59,7 @@ vi.mock("~/db.server", () => ({ { get(_t, prop) { if (prop === holder.REPLICA_BRAND) return true; - const marker = holder.replicaMarker as Record | undefined; - return marker ? marker[prop] : undefined; + return undefined; }, } ), @@ -108,27 +101,8 @@ vi.mock("~/v3/mollifier/syntheticApiResponses.server", () => ({ buildSyntheticTraceBody: (r: unknown) => ({ synthetic: true, run: r }), })); -// Sync-route peripherals (orthogonal to the run read under test). -vi.mock("~/services/session.server", () => ({ - getUserId: async () => holder.userId, -})); -vi.mock("~/v3/runOpsMigration/controlPlaneResolver.server", () => ({ - controlPlaneResolver: { resolveEnv: async () => holder.resolvedEnv }, -})); -vi.mock("~/utils/longPollingFetch", () => ({ - longPollingFetch: async () => - new Response("shape-stream-ok", { status: 200, headers: { "x-longpoll": "hit" } }), -})); -// Ensure ELECTRIC_ORIGIN is defined for the sync route's happy path; keep every other env value real -// so the heavy apiBuilder import graph still loads. -vi.mock("~/env.server", async (importOriginal) => { - const actual = (await importOriginal()) as { env: Record }; - return { env: { ...actual.env, ELECTRIC_ORIGIN: "http://electric.test" } }; -}); - import { loader as spansLoader } from "~/routes/api.v1.runs.$runId.spans.$spanId"; import { loader as traceLoader } from "~/routes/api.v1.runs.$runId.trace"; -import { loader as syncTraceRunsLoader } from "~/routes/sync.traces.runs.$traceId"; // A cuid (25 chars after `run_`) classifies LEGACY, so both the create and the friendlyId/traceId // reads route to the legacy (control-plane) store — the store that owns these runs. @@ -266,13 +240,6 @@ function traceRequest(runId: string) { context: {} as never, }; } -function syncRequest(traceId: string) { - return { - request: new Request(`https://app.trigger.dev/sync/traces/runs/${traceId}?live=true`), - params: { traceId }, - context: {} as never, - } as never; -} describe("run-trace/span-detail route loaders under a lagging replica", () => { // spans loader — findResource findRun ($replica) @@ -466,81 +433,4 @@ describe("run-trace/span-detail route loaders under a lagging replica", () => { expect(body2.trace).toEqual({ rootSpanId: `span_${suffix}`, spans: [] }); } ); - - // sync trace-runs loader — findRun by traceId ($replica), with a primary fallback - heteroRunOpsPostgresTest( - "sync trace-runs loader: a live run lagging the replica is recovered via the primary fallback", - async ({ prisma14, prisma17 }) => { - const suffix = `sync_${seq++}`; - const seed = await seedTenant(prisma14, suffix); - const runId = `run_${CUID_25}`; - const friendlyId = `run_${suffix}`; - const traceId = "a".repeat(32); - const userId = `user_${suffix}`; - - // The dashboard user, joined to the org so the route's real orgMember check passes. - await prisma14.user.create({ - data: { id: userId, email: `u-${suffix}@example.com`, authenticationMethod: "MAGIC_LINK" }, - }); - await prisma14.orgMember.create({ - data: { userId, organizationId: seed.organization.id, role: "ADMIN" }, - }); - - const lagged = buildRouter(prisma14, prisma17, [{ model: "taskRun", mode: "missing" }]); - await lagged.legacyStore.createRun( - buildCreateRunInput({ - runId, - friendlyId, - organizationId: seed.organization.id, - projectId: seed.project.id, - runtimeEnvironmentId: seed.environment.id, - traceId, - spanId: `span_${suffix}`, - }) - ); - - holder.store = lagged.router; - holder.environment = authEnvironment(seed); - holder.userId = userId; - holder.resolvedEnv = { organizationId: seed.organization.id }; - holder.replicaMarker = { orgMember: prisma14.orgMember }; - - const res = (await syncTraceRunsLoader(syncRequest(traceId))) as Response; - - // The frozen replica WAS consulted (the lag was really exercised). - expect(lagged.legacyReplica.wasHit("taskRun")).toBe(true); - - // The primary fallback recovers the live run and the loader proceeds to the shape stream (200). - expect(res.status).toBe(200); - expect(res.headers.get("x-longpoll")).toBe("hit"); - } - ); - - // Negative control for the sync route: a truly-absent run must still 404 (the primary fallback - // recovers a LIVE run, it does not turn every miss into a 200). - heteroRunOpsPostgresTest( - "sync trace-runs loader: 404s when the run is absent on the primary too", - async ({ prisma14, prisma17 }) => { - const suffix = `sync_absent_${seq++}`; - const seed = await seedTenant(prisma14, suffix); - const userId = `user_${suffix}`; - await prisma14.user.create({ - data: { id: userId, email: `u-${suffix}@example.com`, authenticationMethod: "MAGIC_LINK" }, - }); - await prisma14.orgMember.create({ - data: { userId, organizationId: seed.organization.id, role: "ADMIN" }, - }); - - const lagged = buildRouter(prisma14, prisma17, [{ model: "taskRun", mode: "missing" }]); - holder.store = lagged.router; - holder.environment = authEnvironment(seed); - holder.userId = userId; - holder.resolvedEnv = { organizationId: seed.organization.id }; - holder.replicaMarker = { orgMember: prisma14.orgMember }; - - const res = (await syncTraceRunsLoader(syncRequest("b".repeat(32)))) as Response; - expect(lagged.legacyReplica.wasHit("taskRun")).toBe(true); - expect(res.status).toBe(404); - } - ); }); diff --git a/internal-packages/run-store/src/runOpsStore.routesSpanTraceReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.routesSpanTraceReadView.replicaLag.test.ts index 15754d55724..5a5ecb4a790 100644 --- a/internal-packages/run-store/src/runOpsStore.routesSpanTraceReadView.replicaLag.test.ts +++ b/internal-packages/run-store/src/runOpsStore.routesSpanTraceReadView.replicaLag.test.ts @@ -1,5 +1,5 @@ -// Lagging-replica coverage for the SPAN-DETAIL / TRACE / TRACE-SYNC route read views on the run-ops -// split. Four run lookups back three GET routes; NONE is a read-your-writes mutation gate — every one +// Lagging-replica coverage for the SPAN-DETAIL / TRACE route read views on the run-ops +// split. Three run lookups back two GET routes; NONE is a read-your-writes mutation gate — every one // resolves a run for a DISPLAY/GET response and tolerates a stale/missing read by a named caller // mechanism. This builds the store as each route holds it (`runStore` via a RoutingRunStore), freezes // the OWNING store's replica with the shared laggingReplica primitive, invokes each read EXACTLY as @@ -22,8 +22,6 @@ // 2. spans.$spanId findRuns({where:{runtimeEnvironmentId, parentSpanId}, take:50, select}) — open // predicate → both stores' replicas, merged. // 3. trace findRun({friendlyId, runtimeEnvironmentId}) — same shape as read 1. -// 4. sync.traces findRun({traceId}, {select:{runtimeEnvironmentId}}) — unclassifiable where → -// unrouted lookup (NEW-first then LEGACY, both replicas). // // Real split topology via heteroRunOpsPostgresTest — NEVER mocked. The laggingReplica primitive // freezes findFirst/findMany/findUnique on a chosen Prisma model so a replica-routed read misses the @@ -136,7 +134,7 @@ function makeRouters(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { return { laggingRouter, healthyRouter, legacyReplica }; } -describe("run-ops split — span-detail / trace / trace-sync route read views vs. a lagging replica", () => { +describe("run-ops split — span-detail / trace route read views vs. a lagging replica", () => { // --- spans.$spanId findPgRun -------------------------------------------------------------------- // findResource resolves the run for the span-detail GET. Under lag the owning (legacy) replica // returns null; the route falls through to the mollifier buffer fallback and, failing that, returns @@ -273,50 +271,4 @@ describe("run-ops split — span-detail / trace / trace-sync route read views vs expect(recovered!.friendlyId).toBe(friendlyId); } ); - - // --- sync.traces findRun ------------------------------------------------------------------------ - // The dashboard trace-sync loader resolves a run by traceId to authorize the user + resolve the - // env, then long-polls Electric for the "TaskRun" shape. The where carries ONLY traceId - // (unclassifiable) → unrouted lookup (NEW-first then LEGACY, both on their replicas). Under lag the - // owning replica returns null → the loader returns a 404 "No run found". This is a live-sync loader - // the dashboard re-establishes (long-poll); a run reaching it is user-navigation sourced and has - // existed for seconds, so the stale-null window is transient and re-polled. Tolerated (display/sync - // loader, re-polled; no mutation). Store fact under lag: null. - heteroRunOpsPostgresTest( - "sync.traces findRun(traceId, select, branded $replica) is NULL under owning-replica lag; caught-up replica recovers the run (re-polled sync loader tolerates the stale 404)", - async ({ prisma14, prisma17 }) => { - const { laggingRouter, healthyRouter, legacyReplica } = makeRouters(prisma14, prisma17); - const seed = await seedEnvironment(prisma14, "trace_sync"); - const runId = `run_${CUID_25}`; - const traceId = "trace_sync_unique"; - await prisma14.taskRun.create({ - data: taskRunData({ - id: runId, - friendlyId: `run_tracesync00000000000000`, - organizationId: seed.organization.id, - projectId: seed.project.id, - runtimeEnvironmentId: seed.environment.id, - traceId, - spanId: "span_trace_sync", - }), - }); - - // Exactly the call site: findRun({traceId}, {select:{runtimeEnvironmentId}}, $replica). - const stale = await laggingRouter.findRun( - { traceId }, - { select: { runtimeEnvironmentId: true } }, - BRANDED_REPLICA - ); - expect(stale).toBeNull(); - expect(legacyReplica.wasHit()).toBe(true); - - const recovered = (await healthyRouter.findRun( - { traceId }, - { select: { runtimeEnvironmentId: true } }, - BRANDED_REPLICA - )) as { runtimeEnvironmentId: string } | null; - expect(recovered).not.toBeNull(); - expect(recovered!.runtimeEnvironmentId).toBe(seed.environment.id); - } - ); }); From 44eca4d1665e52e09495a57082ab42c7c240ad9a Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Tue, 28 Jul 2026 11:28:09 +0200 Subject: [PATCH 23/67] feat(webapp): org-gated internal API origin in run env vars (#4366) Adds an opt-in way for operators to route deployed runs' API traffic through a different origin than the public one, per organization. Set `INTERNAL_API_ORIGIN` on the webapp and enable the `internalApiOriginEnabled` feature flag (globally or per org, with the org override winning in both directions): deployed runs for enabled orgs then get `TRIGGER_API_URL` set to the internal origin instead of `API_ORIGIN`. Useful for gradually moving run traffic onto a private network path. ## Design The origin is resolved when an attempt starts, so flag changes take effect on the next attempt and roll back the same way, with no task redeploys. The org override is read fresh per attempt; the global default comes from the cached flags registry (a cold read fails safe to the public origin). When `INTERNAL_API_ORIGIN` is unset the flag is a no-op and no extra queries run, so existing deployments are unaffected. Dev runs always use the public origin, and `TRIGGER_STREAM_URL` remains unchanged. --- .server-changes/internal-api-origin-flag.md | 6 ++ apps/webapp/app/env.server.ts | 5 ++ .../api.v1.projects.$projectRef.envvars.ts | 3 + .../environmentVariablesRepository.server.ts | 33 ++++++++- apps/webapp/app/v3/featureFlags.ts | 36 +++++++++ .../worker/workerGroupTokenService.server.ts | 2 + apps/webapp/test/internalApiOrigin.test.ts | 73 +++++++++++++++++++ 7 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 .server-changes/internal-api-origin-flag.md create mode 100644 apps/webapp/test/internalApiOrigin.test.ts diff --git a/.server-changes/internal-api-origin-flag.md b/.server-changes/internal-api-origin-flag.md new file mode 100644 index 00000000000..48b26ba8536 --- /dev/null +++ b/.server-changes/internal-api-origin-flag.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Self-hosted instances can now serve deployed runs' API traffic from a different origin than the public one, per organization, via the `INTERNAL_API_ORIGIN` environment variable and a feature flag. diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 2d28827f7d7..f2240bd6813 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -233,6 +233,11 @@ const EnvironmentSchema = z LOGIN_RATE_LIMITS_ENABLED: BoolEnv.default(true), APP_ORIGIN: z.string().default("http://localhost:3030"), API_ORIGIN: z.string().optional(), + // Alternative API origin for deployed runs whose org has the + // internalApiOriginEnabled feature flag on. Unset = flag is a no-op. + INTERNAL_API_ORIGIN: z.string().optional(), + // Global default for internalApiOriginEnabled when an org hasn't set it. + INTERNAL_API_ORIGIN_ENABLED: z.string().default("0"), STREAM_ORIGIN: z.string().optional(), ELECTRIC_ORIGIN: z.string().default("http://localhost:3060"), // A comma separated list of electric origins to shard into different electric instances by environmentId diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.ts b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.ts index 151c182e166..4fc8b680d1c 100644 --- a/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.ts +++ b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.ts @@ -1,6 +1,7 @@ import { type LoaderFunctionArgs, json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; +import { env } from "~/env.server"; import { authenticateApiRequest } from "~/services/apiAuth.server"; import { resolveVariablesForEnvironment } from "~/v3/environmentVariables/environmentVariablesRepository.server"; @@ -44,6 +45,8 @@ export async function loader({ request, params }: LoaderFunctionArgs) { }, include: { parentEnvironment: true, + // Feeds resolveProdApiOrigin; only loaded when internal-origin routing is possible. + ...(env.INTERNAL_API_ORIGIN ? { organization: { select: { featureFlags: true } } } : {}), }, }); diff --git a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts index 1a0b8247347..c4dd631470d 100644 --- a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts +++ b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts @@ -7,6 +7,8 @@ import { env } from "~/env.server"; import { getSecretStore } from "~/services/secrets/secretStore.server"; import { deduplicateVariableArray } from "../deduplicateVariableArray.server"; import { removeBlacklistedVariables } from "../environmentVariableRules.server"; +import { FEATURE_FLAG, resolveInternalApiOriginEnabled } from "../featureFlags"; +import { globalFlagsRegistry } from "../globalFlagsRegistry.server"; import { generateFriendlyId } from "../friendlyIdentifiers"; import { type CreateEnvironmentVariables, @@ -934,7 +936,7 @@ export type RuntimeEnvironmentForEnvRepo = Pick< | "organizationId" | "branchName" | "builtInEnvironmentVariableOverrides" ->; +> & { organization?: { featureFlags: unknown } | null }; export const environmentVariablesRepository = new EnvironmentVariablesRepository(); @@ -1146,10 +1148,35 @@ async function resolveOverridableOtelDevVariables( return result; } +// Deployed runs normally get the public API origin. When INTERNAL_API_ORIGIN is +// set and the org's internalApiOriginEnabled flag resolves on (org override wins +// in both directions; INTERNAL_API_ORIGIN_ENABLED is the global default applied +// only when the org has not set it), they get the internal origin instead. The +// global default is the cached DB flag with INTERNAL_API_ORIGIN_ENABLED as the +// fallback; org flags are read in-memory, so a flip applies on the next attempt. +function resolveProdApiOrigin(runtimeEnvironment: RuntimeEnvironmentForEnvRepo): string { + const publicOrigin = env.API_ORIGIN ?? env.APP_ORIGIN; + + if (!env.INTERNAL_API_ORIGIN) { + return publicOrigin; + } + + const enabled = resolveInternalApiOriginEnabled({ + orgFeatureFlags: runtimeEnvironment.organization?.featureFlags, + globalDefault: + globalFlagsRegistry.current()?.[FEATURE_FLAG.internalApiOriginEnabled] ?? + env.INTERNAL_API_ORIGIN_ENABLED === "1", + }); + + return enabled ? env.INTERNAL_API_ORIGIN : publicOrigin; +} + async function resolveBuiltInProdVariables( runtimeEnvironment: RuntimeEnvironmentForEnvRepo, parentEnvironment?: RuntimeEnvironmentForEnvRepo ) { + const apiOrigin = resolveProdApiOrigin(runtimeEnvironment); + let result: Array = [ { key: "TRIGGER_SECRET_KEY", @@ -1157,9 +1184,11 @@ async function resolveBuiltInProdVariables( }, { key: "TRIGGER_API_URL", - value: env.API_ORIGIN ?? env.APP_ORIGIN, + value: apiOrigin, }, { + // Deliberately not switched by internalApiOriginEnabled: streams are + // long-lived connections served on their own path. key: "TRIGGER_STREAM_URL", value: env.STREAM_ORIGIN ?? env.API_ORIGIN ?? env.APP_ORIGIN, }, diff --git a/apps/webapp/app/v3/featureFlags.ts b/apps/webapp/app/v3/featureFlags.ts index 49a18eaf35f..22e5541d561 100644 --- a/apps/webapp/app/v3/featureFlags.ts +++ b/apps/webapp/app/v3/featureFlags.ts @@ -12,6 +12,7 @@ export const FEATURE_FLAG = { hasSso: "hasSso", mollifierEnabled: "mollifierEnabled", workerQueueScheduledSplitEnabled: "workerQueueScheduledSplitEnabled", + internalApiOriginEnabled: "internalApiOriginEnabled", realtimeBackend: "realtimeBackend", computeMigrationEnabled: "computeMigrationEnabled", computeMigrationFreePercentage: "computeMigrationFreePercentage", @@ -38,6 +39,12 @@ export const FeatureFlagCatalog = { [FEATURE_FLAG.hasSso]: z.coerce.boolean(), [FEATURE_FLAG.mollifierEnabled]: z.coerce.boolean(), [FEATURE_FLAG.workerQueueScheduledSplitEnabled]: z.coerce.boolean(), + // Routes deployed runs' TRIGGER_API_URL to INTERNAL_API_ORIGIN. Per-org, with + // INTERNAL_API_ORIGIN_ENABLED as the global default (org wins). No-op unless + // INTERNAL_API_ORIGIN is set. + // Strict z.boolean(): coercion turns the string "false" into true, which + // would silently enable the wrong orgs if written as a string. + [FEATURE_FLAG.internalApiOriginEnabled]: z.boolean(), // Which backend serves the realtime run feed. Controllable // globally and per-org (org wins). Defaults to "electric" when unset. // "shadow" serves Electric but diffs the native path in the background. @@ -104,6 +111,35 @@ export function validatePartialFeatureFlags(values: Record) { } // Utility types for catalog-driven UI rendering +/** + * Resolve whether deployed runs should use the internal API origin, from the + * org's feature-flags JSON. Precedence: a per-org override wins in BOTH + * directions; the global default applies only when the org has not set the + * flag (or set it to something invalid). + */ +export function resolveInternalApiOriginEnabled({ + orgFeatureFlags, + globalDefault, +}: { + orgFeatureFlags: unknown; + globalDefault: boolean; +}): boolean { + const override = + orgFeatureFlags && typeof orgFeatureFlags === "object" && !Array.isArray(orgFeatureFlags) + ? (orgFeatureFlags as Record)[FEATURE_FLAG.internalApiOriginEnabled] + : undefined; + + if (override !== undefined) { + const parsed = FeatureFlagCatalog[FEATURE_FLAG.internalApiOriginEnabled].safeParse(override); + + if (parsed.success) { + return parsed.data; + } + } + + return globalDefault; +} + export type FlagControlType = | { type: "boolean" } | { type: "enum"; options: string[] } diff --git a/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts b/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts index ba4741a5ecc..0925c386761 100644 --- a/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts +++ b/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts @@ -544,6 +544,8 @@ export class AuthenticatedWorkerInstance extends WithRunEngine { }, include: { parentEnvironment: true, + // Feeds resolveProdApiOrigin; only loaded when internal-origin routing is possible. + ...(env.INTERNAL_API_ORIGIN ? { organization: { select: { featureFlags: true } } } : {}), }, }); diff --git a/apps/webapp/test/internalApiOrigin.test.ts b/apps/webapp/test/internalApiOrigin.test.ts new file mode 100644 index 00000000000..6971f929609 --- /dev/null +++ b/apps/webapp/test/internalApiOrigin.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; +import { resolveInternalApiOriginEnabled } from "~/v3/featureFlags"; + +describe("resolveInternalApiOriginEnabled", () => { + it("returns the global default when the org has no flags", () => { + expect(resolveInternalApiOriginEnabled({ orgFeatureFlags: null, globalDefault: false })).toBe( + false + ); + expect(resolveInternalApiOriginEnabled({ orgFeatureFlags: null, globalDefault: true })).toBe( + true + ); + expect(resolveInternalApiOriginEnabled({ orgFeatureFlags: {}, globalDefault: true })).toBe( + true + ); + }); + + it("lets an org override win in both directions", () => { + expect( + resolveInternalApiOriginEnabled({ + orgFeatureFlags: { internalApiOriginEnabled: true }, + globalDefault: false, + }) + ).toBe(true); + + expect( + resolveInternalApiOriginEnabled({ + orgFeatureFlags: { internalApiOriginEnabled: false }, + globalDefault: true, + }) + ).toBe(false); + }); + + it("ignores invalid overrides and falls back to the global default", () => { + // Strict z.boolean(): the string "false" must not coerce to an enable. + expect( + resolveInternalApiOriginEnabled({ + orgFeatureFlags: { internalApiOriginEnabled: "false" }, + globalDefault: false, + }) + ).toBe(false); + + expect( + resolveInternalApiOriginEnabled({ + orgFeatureFlags: { internalApiOriginEnabled: "true" }, + globalDefault: false, + }) + ).toBe(false); + + expect( + resolveInternalApiOriginEnabled({ + orgFeatureFlags: { internalApiOriginEnabled: 1 }, + globalDefault: false, + }) + ).toBe(false); + + // The fallback must follow the global default, not hardcode false. + expect( + resolveInternalApiOriginEnabled({ + orgFeatureFlags: { internalApiOriginEnabled: "false" }, + globalDefault: true, + }) + ).toBe(true); + }); + + it("ignores non-object flag containers", () => { + expect(resolveInternalApiOriginEnabled({ orgFeatureFlags: [], globalDefault: true })).toBe( + true + ); + expect(resolveInternalApiOriginEnabled({ orgFeatureFlags: "junk", globalDefault: false })).toBe( + false + ); + }); +}); From 38bf82aebe363538215da015fbc47b1856039989 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Tue, 28 Jul 2026 14:23:41 +0100 Subject: [PATCH 24/67] feat(cli,webapp): target notifications by minimum CLI version (#4407) --- .changeset/target-cli-notifications.md | 5 + .../webapp/app/routes/admin.notifications.tsx | 30 ++- .../routes/api.v1.platform-notifications.ts | 2 + .../services/platformNotificationSchemas.ts | 228 +++++++++++++++++ .../platformNotificationVersionTargeting.ts | 29 +++ .../services/platformNotifications.server.ts | 238 ++---------------- apps/webapp/package.json | 2 + .../webapp/test/platformNotifications.test.ts | 100 ++++++++ packages/cli-v3/src/apiClient.ts | 1 + pnpm-lock.yaml | 25 +- 10 files changed, 430 insertions(+), 230 deletions(-) create mode 100644 .changeset/target-cli-notifications.md create mode 100644 apps/webapp/app/services/platformNotificationSchemas.ts create mode 100644 apps/webapp/app/services/platformNotificationVersionTargeting.ts create mode 100644 apps/webapp/test/platformNotifications.test.ts diff --git a/.changeset/target-cli-notifications.md b/.changeset/target-cli-notifications.md new file mode 100644 index 00000000000..bdb6f7240c0 --- /dev/null +++ b/.changeset/target-cli-notifications.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Send the running CLI version when checking for platform notifications so notices can be limited to compatible CLI releases. diff --git a/apps/webapp/app/routes/admin.notifications.tsx b/apps/webapp/app/routes/admin.notifications.tsx index 42e02afa6cc..8e9e4beb374 100644 --- a/apps/webapp/app/routes/admin.notifications.tsx +++ b/apps/webapp/app/routes/admin.notifications.tsx @@ -140,6 +140,8 @@ function parseNotificationFormData(formData: FormData) { const cliShowEvery = formData.get("cliShowEvery") ? Number(formData.get("cliShowEvery")) : undefined; + const minimumCliVersion = + (formData.get("minimumCliVersion") as string | null)?.trim() || undefined; const discoveryFilePatterns = (formData.get("discoveryFilePatterns") as string) || ""; const discoveryContentPattern = (formData.get("discoveryContentPattern") as string) || undefined; @@ -177,6 +179,7 @@ function parseNotificationFormData(formData: FormData) { cliMaxShowCount, cliMaxDaysAfterFirstSeen, cliShowEvery, + minimumCliVersion, discovery, }; } @@ -192,6 +195,9 @@ function buildPayloadInput(fields: ReturnType) ...(fields.image ? { image: fields.image } : {}), ...(fields.dismissOnAction ? { dismissOnAction: true } : {}), ...(fields.discovery ? { discovery: fields.discovery } : {}), + ...(fields.surface === "CLI" && fields.minimumCliVersion + ? { minimumCliVersion: fields.minimumCliVersion } + : {}), }, }; } @@ -669,6 +675,7 @@ type NotificationFormDefaults = { contentPattern?: string; matchBehavior: "show-if-found" | "show-if-not-found"; } | null; + payloadMinimumCliVersion?: string | null; cliMaxShowCount?: number | null; cliMaxDaysAfterFirstSeen?: number | null; cliShowEvery?: number | null; @@ -1024,7 +1031,19 @@ function NotificationForm({ <> -
+
+
+ + + Exact SemVer, inclusive +

CLI Settings

+ {n.payloadMinimumCliVersion && ( + + )} {n.cliMaxShowCount != null && ( )} diff --git a/apps/webapp/app/routes/api.v1.platform-notifications.ts b/apps/webapp/app/routes/api.v1.platform-notifications.ts index 7299c407383..62e3b000f9f 100644 --- a/apps/webapp/app/routes/api.v1.platform-notifications.ts +++ b/apps/webapp/app/routes/api.v1.platform-notifications.ts @@ -12,10 +12,12 @@ export async function loader({ request }: LoaderFunctionArgs) { const url = new URL(request.url); const projectRef = url.searchParams.get("projectRef") ?? undefined; + const cliVersion = request.headers.get("x-trigger-cli-version")?.trim() || undefined; const notification = await getNextCliNotification({ userId: authenticationResult.userId, projectRef, + cliVersion, }); return json({ notification }); diff --git a/apps/webapp/app/services/platformNotificationSchemas.ts b/apps/webapp/app/services/platformNotificationSchemas.ts new file mode 100644 index 00000000000..a91521e07a5 --- /dev/null +++ b/apps/webapp/app/services/platformNotificationSchemas.ts @@ -0,0 +1,228 @@ +import { z } from "zod"; +import { normalizeExactSemVer } from "./platformNotificationVersionTargeting"; + +const DiscoverySchema = z.object({ + filePatterns: z.array(z.string().min(1)).min(1), + contentPattern: z + .string() + .max(200) + .optional() + .refine( + (val) => { + if (!val) return true; + try { + new RegExp(val); + return true; + } catch { + return false; + } + }, + { message: "contentPattern must be a valid regular expression" } + ), + matchBehavior: z.enum(["show-if-found", "show-if-not-found"]), +}); + +// Constrain URL fields to http/https; `.url()` alone accepts other schemes +// that would be unsafe to render into an ``. +const httpUrl = z + .string() + .url() + .refine( + (v) => { + try { + const proto = new URL(v).protocol; + return proto === "http:" || proto === "https:"; + } catch { + return false; + } + }, + { message: "URL must use http or https" } + ); + +const ExactSemVerSchema = z + .string() + .transform((value) => value.trim()) + .refine((value) => normalizeExactSemVer(value) !== null, { + message: "minimumCliVersion must be a complete, exact SemVer", + }) + .transform((value) => normalizeExactSemVer(value)!); + +const CardDataV1Schema = z.object({ + type: z.enum(["card", "info", "warn", "error", "success", "changelog"]), + title: z.string(), + description: z.string(), + image: httpUrl.optional(), + actionLabel: z.string().optional(), + actionUrl: httpUrl.optional(), + dismissOnAction: z.boolean().optional(), + discovery: DiscoverySchema.optional(), + minimumCliVersion: ExactSemVerSchema.optional(), +}); + +export const PayloadV1Schema = z.object({ + version: z.literal("1"), + data: CardDataV1Schema, +}); + +export type PayloadV1 = z.infer; + +const SCOPE_REQUIRED_FK: Record = { + USER: "userId", + ORGANIZATION: "organizationId", + PROJECT: "projectId", +}; + +const ALL_FK_FIELDS = ["userId", "organizationId", "projectId"] as const; +const CLI_ONLY_FIELDS = ["cliMaxDaysAfterFirstSeen", "cliMaxShowCount", "cliShowEvery"] as const; + +const NotificationBaseFields = { + title: z.string().min(1), + payload: PayloadV1Schema, + surface: z.enum(["WEBAPP", "CLI"]), + scope: z.enum(["USER", "PROJECT", "ORGANIZATION", "GLOBAL"]), + userId: z.string().optional(), + organizationId: z.string().optional(), + projectId: z.string().optional(), + endsAt: z + .string() + .datetime() + .transform((s) => new Date(s)), + priority: z.number().int().default(0), + cliMaxDaysAfterFirstSeen: z.number().int().positive().optional(), + cliMaxShowCount: z.number().int().positive().optional(), + cliShowEvery: z.number().int().min(2).optional(), +}; + +export const CreatePlatformNotificationSchema = z + .object({ + ...NotificationBaseFields, + startsAt: z + .string() + .datetime() + .transform((s) => new Date(s)) + .optional(), + }) + .superRefine((data, ctx) => { + validateScopeForeignKeys(data, ctx); + validateSurfaceFields(data, ctx); + validatePayloadTypeForSurface(data, ctx); + validateStartsAt(data, ctx); + validateEndsAt(data, ctx); + }); + +function validateScopeForeignKeys( + data: { scope: string; userId?: string; organizationId?: string; projectId?: string }, + ctx: z.RefinementCtx +) { + const requiredFk = SCOPE_REQUIRED_FK[data.scope]; + + if (requiredFk && !data[requiredFk]) { + ctx.addIssue({ + code: "custom", + message: `${requiredFk} is required when scope is ${data.scope}`, + path: [requiredFk], + }); + } + + const forbiddenFks = ALL_FK_FIELDS.filter((fk) => fk !== requiredFk); + for (const fk of forbiddenFks) { + if (data[fk]) { + ctx.addIssue({ + code: "custom", + message: `${fk} must not be set when scope is ${data.scope}`, + path: [fk], + }); + } + } +} + +function validateSurfaceFields( + data: { + surface: string; + payload: PayloadV1; + cliMaxDaysAfterFirstSeen?: number; + cliMaxShowCount?: number; + cliShowEvery?: number; + }, + ctx: z.RefinementCtx +) { + if (data.surface !== "WEBAPP") return; + + for (const field of CLI_ONLY_FIELDS) { + if (data[field] !== undefined) { + ctx.addIssue({ + code: "custom", + message: `${field} is not allowed for WEBAPP surface`, + path: [field], + }); + } + } + + if (data.payload.data.minimumCliVersion !== undefined) { + ctx.addIssue({ + code: "custom", + message: "minimumCliVersion is not allowed for WEBAPP surface", + path: ["payload", "data", "minimumCliVersion"], + }); + } +} + +function validateStartsAt(data: { startsAt?: Date }, ctx: z.RefinementCtx) { + if (!data.startsAt) return; + + const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000); + if (data.startsAt < oneHourAgo) { + ctx.addIssue({ + code: "custom", + message: "startsAt must be within the last hour or in the future", + path: ["startsAt"], + }); + } +} + +const CLI_TYPES = new Set(["info", "warn", "error", "success"]); +const WEBAPP_TYPES = new Set(["card", "changelog"]); + +function validatePayloadTypeForSurface( + data: { surface: string; payload: PayloadV1 }, + ctx: z.RefinementCtx +) { + const allowedTypes = data.surface === "CLI" ? CLI_TYPES : WEBAPP_TYPES; + if (!allowedTypes.has(data.payload.data.type)) { + ctx.addIssue({ + code: "custom", + message: `payload.data.type "${data.payload.data.type}" is not allowed for ${data.surface} surface`, + path: ["payload", "data", "type"], + }); + } +} + +function validateEndsAt(data: { startsAt?: Date; endsAt: Date }, ctx: z.RefinementCtx) { + const effectiveStart = data.startsAt ?? new Date(); + if (data.endsAt <= effectiveStart) { + ctx.addIssue({ + code: "custom", + message: "endsAt must be after startsAt", + path: ["endsAt"], + }); + } +} + +export type CreatePlatformNotificationInput = z.input; + +export const UpdatePlatformNotificationSchema = z + .object({ + ...NotificationBaseFields, + id: z.string().min(1), + startsAt: z + .string() + .datetime() + .transform((s) => new Date(s)), + }) + .superRefine((data, ctx) => { + validateScopeForeignKeys(data, ctx); + validateSurfaceFields(data, ctx); + validatePayloadTypeForSurface(data, ctx); + // Existing notifications may have a startsAt in the past. + validateEndsAt(data, ctx); + }); diff --git a/apps/webapp/app/services/platformNotificationVersionTargeting.ts b/apps/webapp/app/services/platformNotificationVersionTargeting.ts new file mode 100644 index 00000000000..30fbe001c11 --- /dev/null +++ b/apps/webapp/app/services/platformNotificationVersionTargeting.ts @@ -0,0 +1,29 @@ +import * as semver from "semver"; + +export function normalizeExactSemVer(value: string): string | null { + const trimmed = value.trim(); + const parsed = semver.parse(trimmed); + + if (!parsed) return null; + + const normalized = `${parsed.version}${ + parsed.build.length > 0 ? `+${parsed.build.join(".")}` : "" + }`; + + return normalized === trimmed ? normalized : null; +} + +export function isCliVersionEligible( + minimumCliVersion: string | undefined, + cliVersion: string | undefined +): boolean { + if (minimumCliVersion === undefined) return true; + + const normalizedMinimum = normalizeExactSemVer(minimumCliVersion); + if (!normalizedMinimum || cliVersion === undefined) return false; + + const normalizedCliVersion = normalizeExactSemVer(cliVersion); + if (!normalizedCliVersion) return false; + + return semver.gte(normalizedCliVersion, normalizedMinimum); +} diff --git a/apps/webapp/app/services/platformNotifications.server.ts b/apps/webapp/app/services/platformNotifications.server.ts index fa2e0ea8429..24e95740a70 100644 --- a/apps/webapp/app/services/platformNotifications.server.ts +++ b/apps/webapp/app/services/platformNotifications.server.ts @@ -1,4 +1,4 @@ -import { z } from "zod"; +import type { z } from "zod"; import { errAsync, fromPromise, type ResultAsync } from "neverthrow"; import { prisma } from "~/db.server"; import { @@ -6,64 +6,20 @@ import { type PlatformNotificationSurface, } from "@trigger.dev/database"; import { incrementCliRequestCounter } from "./platformNotificationCounter.server"; - -// --- Payload schema (spec v1) --- - -const DiscoverySchema = z.object({ - filePatterns: z.array(z.string().min(1)).min(1), - contentPattern: z - .string() - .max(200) - .optional() - .refine( - (val) => { - if (!val) return true; - try { - new RegExp(val); - return true; - } catch { - return false; - } - }, - { message: "contentPattern must be a valid regular expression" } - ), - matchBehavior: z.enum(["show-if-found", "show-if-not-found"]), -}); - -// Constrain URL fields to http/https; `.url()` alone accepts other schemes -// that would be unsafe to render into an ``. -const httpUrl = z - .string() - .url() - .refine( - (v) => { - try { - const proto = new URL(v).protocol; - return proto === "http:" || proto === "https:"; - } catch { - return false; - } - }, - { message: "URL must use http or https" } - ); - -const CardDataV1Schema = z.object({ - type: z.enum(["card", "info", "warn", "error", "success", "changelog"]), - title: z.string(), - description: z.string(), - image: httpUrl.optional(), - actionLabel: z.string().optional(), - actionUrl: httpUrl.optional(), - dismissOnAction: z.boolean().optional(), - discovery: DiscoverySchema.optional(), -}); - -const PayloadV1Schema = z.object({ - version: z.literal("1"), - data: CardDataV1Schema, -}); - -export type PayloadV1 = z.infer; +import { + CreatePlatformNotificationSchema, + type CreatePlatformNotificationInput, + type PayloadV1, + PayloadV1Schema, + UpdatePlatformNotificationSchema, +} from "./platformNotificationSchemas"; +import { isCliVersionEligible } from "./platformNotificationVersionTargeting"; + +export { + CreatePlatformNotificationSchema, + UpdatePlatformNotificationSchema, +} from "./platformNotificationSchemas"; +export type { CreatePlatformNotificationInput, PayloadV1 } from "./platformNotificationSchemas"; export type PlatformNotificationWithPayload = { id: string; @@ -136,6 +92,9 @@ export async function getAdminNotificationsList({ ? (parsed.data.data.dismissOnAction ?? false) : false, payloadDiscovery: parsed.success ? (parsed.data.data.discovery ?? null) : null, + payloadMinimumCliVersion: parsed.success + ? (parsed.data.data.minimumCliVersion ?? null) + : null, cliMaxShowCount: n.cliMaxShowCount, cliMaxDaysAfterFirstSeen: n.cliMaxDaysAfterFirstSeen, cliShowEvery: n.cliShowEvery, @@ -435,9 +394,11 @@ function isCliNotificationExpired( export async function getNextCliNotification({ userId, projectRef, + cliVersion, }: { userId: string; projectRef?: string; + cliVersion?: string; }): Promise<{ id: string; payload: PayloadV1; @@ -520,10 +481,11 @@ export async function getNextCliNotification({ const interaction = n.interactions[0] ?? null; if (interaction?.cliDismissedAt) continue; - if (isCliNotificationExpired(interaction, n)) continue; const parsed = PayloadV1Schema.safeParse(n.payload); if (!parsed.success) continue; + if (!isCliVersionEligible(parsed.data.data.minimumCliVersion, cliVersion)) continue; + if (isCliNotificationExpired(interaction, n)) continue; // Check cliShowEvery using the global request counter if (n.cliShowEvery !== null && requestCounter % n.cliShowEvery !== 0) { @@ -562,161 +524,7 @@ export async function getNextCliNotification({ return null; } -// --- Create: admin endpoint support --- - -const SCOPE_REQUIRED_FK: Record = { - USER: "userId", - ORGANIZATION: "organizationId", - PROJECT: "projectId", -}; - -const ALL_FK_FIELDS = ["userId", "organizationId", "projectId"] as const; -const CLI_ONLY_FIELDS = ["cliMaxDaysAfterFirstSeen", "cliMaxShowCount", "cliShowEvery"] as const; - -const NotificationBaseFields = { - title: z.string().min(1), - payload: PayloadV1Schema, - surface: z.enum(["WEBAPP", "CLI"]), - scope: z.enum(["USER", "PROJECT", "ORGANIZATION", "GLOBAL"]), - userId: z.string().optional(), - organizationId: z.string().optional(), - projectId: z.string().optional(), - endsAt: z - .string() - .datetime() - .transform((s) => new Date(s)), - priority: z.number().int().default(0), - cliMaxDaysAfterFirstSeen: z.number().int().positive().optional(), - cliMaxShowCount: z.number().int().positive().optional(), - cliShowEvery: z.number().int().min(2).optional(), -}; - -export const CreatePlatformNotificationSchema = z - .object({ - ...NotificationBaseFields, - startsAt: z - .string() - .datetime() - .transform((s) => new Date(s)) - .optional(), - }) - .superRefine((data, ctx) => { - validateScopeForeignKeys(data, ctx); - validateSurfaceFields(data, ctx); - validatePayloadTypeForSurface(data, ctx); - validateStartsAt(data, ctx); - validateEndsAt(data, ctx); - }); - -function validateScopeForeignKeys( - data: { scope: string; userId?: string; organizationId?: string; projectId?: string }, - ctx: z.RefinementCtx -) { - const requiredFk = SCOPE_REQUIRED_FK[data.scope]; - - if (requiredFk && !data[requiredFk]) { - ctx.addIssue({ - code: "custom", - message: `${requiredFk} is required when scope is ${data.scope}`, - path: [requiredFk], - }); - } - - const forbiddenFks = ALL_FK_FIELDS.filter((fk) => fk !== requiredFk); - for (const fk of forbiddenFks) { - if (data[fk]) { - ctx.addIssue({ - code: "custom", - message: `${fk} must not be set when scope is ${data.scope}`, - path: [fk], - }); - } - } -} - -function validateSurfaceFields( - data: { - surface: string; - cliMaxDaysAfterFirstSeen?: number; - cliMaxShowCount?: number; - cliShowEvery?: number; - }, - ctx: z.RefinementCtx -) { - if (data.surface !== "WEBAPP") return; - - for (const field of CLI_ONLY_FIELDS) { - if (data[field] !== undefined) { - ctx.addIssue({ - code: "custom", - message: `${field} is not allowed for WEBAPP surface`, - path: [field], - }); - } - } -} - -function validateStartsAt(data: { startsAt?: Date }, ctx: z.RefinementCtx) { - if (!data.startsAt) return; - - const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000); - if (data.startsAt < oneHourAgo) { - ctx.addIssue({ - code: "custom", - message: "startsAt must be within the last hour or in the future", - path: ["startsAt"], - }); - } -} - -const CLI_TYPES = new Set(["info", "warn", "error", "success"]); -const WEBAPP_TYPES = new Set(["card", "changelog"]); - -function validatePayloadTypeForSurface( - data: { surface: string; payload: PayloadV1 }, - ctx: z.RefinementCtx -) { - const allowedTypes = data.surface === "CLI" ? CLI_TYPES : WEBAPP_TYPES; - if (!allowedTypes.has(data.payload.data.type)) { - ctx.addIssue({ - code: "custom", - message: `payload.data.type "${data.payload.data.type}" is not allowed for ${data.surface} surface`, - path: ["payload", "data", "type"], - }); - } -} - -function validateEndsAt(data: { startsAt?: Date; endsAt: Date }, ctx: z.RefinementCtx) { - const effectiveStart = data.startsAt ?? new Date(); - if (data.endsAt <= effectiveStart) { - ctx.addIssue({ - code: "custom", - message: "endsAt must be after startsAt", - path: ["endsAt"], - }); - } -} - -export type CreatePlatformNotificationInput = z.input; - -// --- Update: admin endpoint support --- - -export const UpdatePlatformNotificationSchema = z - .object({ - ...NotificationBaseFields, - id: z.string().min(1), - startsAt: z - .string() - .datetime() - .transform((s) => new Date(s)), - }) - .superRefine((data, ctx) => { - validateScopeForeignKeys(data, ctx); - validateSurfaceFields(data, ctx); - validatePayloadTypeForSurface(data, ctx); - // NOTE: No validateStartsAt — existing notifications may have past startsAt - validateEndsAt(data, ctx); - }); +// --- Create and update: admin endpoint support --- type CreateError = { type: "validation"; issues: z.ZodIssue[] } | { type: "db"; message: string }; diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 0eb149f77ed..8819f322f93 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -196,6 +196,7 @@ "remix-auth-google": "^2.0.0", "remix-typedjson": "0.3.1", "remix-utils": "^7.7.0", + "semver": "^7.5.0", "simplur": "^3.0.1", "slug": "^6.0.0", "socket.io": "4.7.4", @@ -242,6 +243,7 @@ "@types/react": "18.2.69", "@types/react-dom": "18.2.7", "@types/regression": "^2.0.6", + "@types/semver": "^7.5.0", "@types/slug": "^5.0.3", "@types/supertest": "^6.0.2", "@types/tar": "^6.1.4", diff --git a/apps/webapp/test/platformNotifications.test.ts b/apps/webapp/test/platformNotifications.test.ts new file mode 100644 index 00000000000..b7b4634339c --- /dev/null +++ b/apps/webapp/test/platformNotifications.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from "vitest"; +import { CreatePlatformNotificationSchema } from "~/services/platformNotificationSchemas"; +import { isCliVersionEligible } from "~/services/platformNotificationVersionTargeting"; + +function createNotificationInput({ + surface = "CLI", + minimumCliVersion, +}: { + surface?: "CLI" | "WEBAPP"; + minimumCliVersion?: string; +} = {}) { + return { + title: "Notification", + surface, + scope: "GLOBAL" as const, + endsAt: new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(), + payload: { + version: "1" as const, + data: { + type: surface === "CLI" ? ("info" as const) : ("card" as const), + title: "Update available", + description: "Update the CLI", + ...(minimumCliVersion === undefined ? {} : { minimumCliVersion }), + }, + }, + }; +} + +describe("platform notification minimum CLI version schema", () => { + it("keeps existing unrestricted CLI payloads valid", () => { + const result = CreatePlatformNotificationSchema.safeParse(createNotificationInput()); + + expect(result.success).toBe(true); + }); + + it("trims and accepts a complete exact SemVer", () => { + const result = CreatePlatformNotificationSchema.parse( + createNotificationInput({ minimumCliVersion: " 4.5.7-beta.1+build.2 " }) + ); + + expect(result.payload.data.minimumCliVersion).toBe("4.5.7-beta.1+build.2"); + }); + + it.each(["4.5", ">=4.5.7", "4.5.x", "v4.5.7", "4.5.7 or newer", "01.2.3"])( + "rejects non-exact SemVer %s", + (minimumCliVersion) => { + const result = CreatePlatformNotificationSchema.safeParse( + createNotificationInput({ minimumCliVersion }) + ); + + expect(result.success).toBe(false); + } + ); + + it("rejects minimumCliVersion for WEBAPP notifications", () => { + const result = CreatePlatformNotificationSchema.safeParse( + createNotificationInput({ surface: "WEBAPP", minimumCliVersion: "4.5.7" }) + ); + + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.issues).toContainEqual( + expect.objectContaining({ path: ["payload", "data", "minimumCliVersion"] }) + ); + } + }); +}); + +describe("CLI notification version eligibility", () => { + it("allows unrestricted notifications without a request version", () => { + expect(isCliVersionEligible(undefined, undefined)).toBe(true); + }); + + it.each([undefined, "", "4.5", "not-a-version"])( + "rejects a targeted notification for missing or malformed request version %s", + (cliVersion) => { + expect(isCliVersionEligible("4.5.7", cliVersion)).toBe(false); + } + ); + + it("fails closed for an invalid stored minimum version", () => { + expect(isCliVersionEligible(">=4.5.7", "4.6.0")).toBe(false); + }); + + it.each([ + ["4.5.6", false], + ["4.5.7-beta.1", false], + ["4.5.7", true], + ["4.5.7+build.9", true], + ["4.6.0", true], + ])("compares %s against inclusive minimum 4.5.7", (cliVersion, expected) => { + expect(isCliVersionEligible("4.5.7", cliVersion)).toBe(expected); + }); + + it("uses SemVer precedence for prerelease minimums", () => { + expect(isCliVersionEligible("4.5.7-beta.2", "4.5.7-beta.1")).toBe(false); + expect(isCliVersionEligible("4.5.7-beta.2", "4.5.7-beta.2+build.1")).toBe(true); + expect(isCliVersionEligible("4.5.7-beta.2", "4.5.7")).toBe(true); + }); +}); diff --git a/packages/cli-v3/src/apiClient.ts b/packages/cli-v3/src/apiClient.ts index d01062440e9..2b76ae021e1 100644 --- a/packages/cli-v3/src/apiClient.ts +++ b/packages/cli-v3/src/apiClient.ts @@ -606,6 +606,7 @@ export class CliApiClient { headers: { Authorization: `Bearer ${this.accessToken}`, "Content-Type": "application/json", + "x-trigger-cli-version": VERSION, }, signal, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2849a27dd58..486a49560f3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -716,6 +716,9 @@ importers: remix-utils: specifier: ^7.7.0 version: 7.7.0(@remix-run/node@2.17.5(typescript@6.0.3))(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/router@1.23.3(patch_hash=33e2966e9ef36aa09955cec9922c5063227b83429157a71ba226369a08627ac9))(crypto-js@4.2.0)(intl-parse-accept-language@1.0.0)(react@18.3.1)(zod@3.25.76) + semver: + specifier: ^7.5.0 + version: 7.8.1 simplur: specifier: ^3.0.1 version: 3.0.1 @@ -849,6 +852,9 @@ importers: '@types/regression': specifier: ^2.0.6 version: 2.0.6 + '@types/semver': + specifier: ^7.5.0 + version: 7.5.1 '@types/slug': specifier: ^5.0.3 version: 5.0.3 @@ -13958,11 +13964,6 @@ packages: engines: {node: '>=10'} hasBin: true - semver@7.7.3: - resolution: {integrity: sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==} - engines: {node: '>=10'} - hasBin: true - semver@7.8.1: resolution: {integrity: sha512-rkVq3IXh+4FDGch+KwzX3aV9W3kO54GyEgpvBzSyctDA6Xtd7RJQV1xmXbeQp5v7+VzLOfVqiutSE6GICgPFvg==} engines: {node: '>=10'} @@ -17525,7 +17526,7 @@ snapshots: outdent: 0.5.0 prettier: 2.8.8 resolve-from: 5.0.0 - semver: 7.7.3 + semver: 7.8.1 '@changesets/assemble-release-plan@5.2.4(patch_hash=a7a12643e8d89a00d5322f750c292e8567b5ee0fc9c613a238a1184c25533e4b)': dependencies: @@ -17534,7 +17535,7 @@ snapshots: '@changesets/get-dependents-graph': 1.3.6 '@changesets/types': 5.2.1 '@manypkg/get-packages': 1.1.3 - semver: 7.7.3 + semver: 7.8.1 '@changesets/changelog-git@0.1.14': dependencies: @@ -17596,7 +17597,7 @@ snapshots: '@manypkg/get-packages': 1.1.3 chalk: 2.4.2 fs-extra: 7.0.1 - semver: 7.7.3 + semver: 7.8.1 '@changesets/get-github-info@0.5.2(encoding@0.1.13)': dependencies: @@ -24149,7 +24150,7 @@ snapshots: postcss-modules-scope: 3.1.1(postcss@8.5.10) postcss-modules-values: 4.0.0(postcss@8.5.10) postcss-value-parser: 4.2.0 - semver: 7.6.3 + semver: 7.8.1 optionalDependencies: webpack: 5.102.1(@swc/core@1.3.26)(esbuild@0.15.18) @@ -25570,7 +25571,7 @@ snapshots: process-warning: 5.0.0 rfdc: 1.4.1 secure-json-parse: 4.0.0 - semver: 7.7.3 + semver: 7.8.1 toad-cache: 3.7.0 fastq@1.15.0: @@ -28690,7 +28691,7 @@ snapshots: cosmiconfig: 9.0.0(typescript@6.0.3) jiti: 1.21.0 postcss: 8.5.15 - semver: 7.6.3 + semver: 7.8.1 optionalDependencies: webpack: 5.102.1(@swc/core@1.3.26)(esbuild@0.15.18) transitivePeerDependencies: @@ -29763,8 +29764,6 @@ snapshots: semver@7.6.3: {} - semver@7.7.3: {} - semver@7.8.1: {} send@0.18.0: From 205bdc31036dc7af8c9eb06c22f2b8e311ac0200 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 28 Jul 2026 15:20:00 +0100 Subject: [PATCH 25/67] docs(wait): separate compute billing from concurrency release (#4405) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wait docs describe the 5 second compute-billing threshold as if it were also the suspension threshold. It isn't, and the gap is confusing when you're sizing a poll interval: - **Compute** stops being charged for any wait longer than 5 seconds. - **Concurrency** is only released once the machine has been snapshotted and shut down. For `wait.for` and `wait.until` that happens 60 seconds into the wait — a shorter wait stays `EXECUTING` and holds its concurrency slot for the whole wait, even though the compute is free. So `await wait.for({ seconds: 30 })` in a polling loop never releases its slot, which looks like a bug if the docs told you waits over 5 seconds checkpoint. ## Changes **`docs/snippets/paused-execution-free.mdx`** — rendered on `/wait`, `/wait-for` and `/wait-until`. Drops "we checkpoint and" from the billing sentence so it's purely about compute, then adds one paragraph for the concurrency half. **`docs/queue-concurrency.mdx`** — the "Waits and concurrency" section states flatly that waiting runs don't consume slots. Adds a short subsection for the time-based exception. **`docs/how-to-reduce-your-spend.mdx`** — "Waits longer than 5 seconds automatically checkpoint your task, meaning you don't pay for compute" → the compute claim only. Code comments follow, plus a pointer that waiting doesn't always free concurrency. **`docs/how-it-works.mdx`** — the Checkpoint-Resume walkthrough used `wait.for({ seconds: 30 })` as *the* example of a wait that suspends. Bumped to 5 minutes and noted the sub-60s exception. No behaviour change — docs only. --- docs/how-it-works.mdx | 6 +++--- docs/how-to-reduce-your-spend.mdx | 8 ++++---- docs/queue-concurrency.mdx | 4 ++++ docs/snippets/paused-execution-free.mdx | 4 +++- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/how-it-works.mdx b/docs/how-it-works.mdx index 88bb4c07036..c7d3e089c1f 100644 --- a/docs/how-it-works.mdx +++ b/docs/how-it-works.mdx @@ -148,7 +148,7 @@ Here's how the Checkpoint-Resume System works: 2. **Subtask Handling**: If a task needs to trigger a subtask, it can do so and wait for its completion using `triggerAndWait` -3. **State Checkpointing**: While waiting for a subtask or during a programmed pause (e.g., `wait.for({ seconds: 30 })`), the system uses CRIU (Checkpoint/Restore In Userspace) to create a checkpoint of the task's entire state, including memory, CPU registers, and open file descriptors. +3. **State Checkpointing**: While waiting for a subtask or during a long programmed pause (e.g., `wait.for({ minutes: 5 })`), the system uses CRIU (Checkpoint/Restore In Userspace) to create a checkpoint of the task's entire state, including memory, CPU registers, and open file descriptors. A `wait.for()` or `wait.until()` shorter than 60 seconds doesn't checkpoint — see [Wait](/wait). 4. **Resource Release**: After checkpointing, the parent task's resources are released, freeing up the execution environment. @@ -178,9 +178,9 @@ export const parentTask = task({ console.log("Child task result:", result); // This will also cause the task to be checkpointed and suspended - await wait.for({ seconds: 30 }); + await wait.for({ minutes: 5 }); - console.log("Resumed after 30 seconds"); + console.log("Resumed after 5 minutes"); return "Parent task completed"; }, diff --git a/docs/how-to-reduce-your-spend.mdx b/docs/how-to-reduce-your-spend.mdx index 7376ffafe74..b19bdb8476b 100644 --- a/docs/how-to-reduce-your-spend.mdx +++ b/docs/how-to-reduce-your-spend.mdx @@ -170,7 +170,7 @@ export const boundedTask = task({ ## Use waitpoints instead of polling -Waits longer than 5 seconds automatically checkpoint your task, meaning you don't pay for compute while waiting. Use `wait.for()`, `wait.until()`, or `triggerAndWait()` instead of polling loops. +You don't pay for compute during any wait longer than 5 seconds. Use `wait.for()`, `wait.until()`, or `triggerAndWait()` instead of polling loops. ```ts import { task, wait } from "@trigger.dev/sdk"; @@ -178,17 +178,17 @@ import { task, wait } from "@trigger.dev/sdk"; export const waitpointTask = task({ id: "waitpoint-task", run: async (payload) => { - // This wait is free - your task is checkpointed + // You aren't charged for compute during this wait await wait.for({ minutes: 5 }); - // Parent is also checkpointed while waiting for child tasks + // The parent isn't charged either while it waits for a child task const result = await childTask.triggerAndWait({ data: payload }); return result; }, }); ``` -[Read more about waitpoints](/wait-for). +Waiting saves you compute, but it doesn't always free up concurrency — see [Wait](/wait). ## Use debounce to consolidate multiple triggers diff --git a/docs/queue-concurrency.mdx b/docs/queue-concurrency.mdx index 24e77f4e67f..b832ffc26da 100644 --- a/docs/queue-concurrency.mdx +++ b/docs/queue-concurrency.mdx @@ -199,6 +199,10 @@ For example, if you have a queue with a `concurrencyLimit` of 1: - When the executing run reaches a waitpoint and checkpoints, it releases its slot - The next queued run can then begin execution +### Short time-based waits keep their slot + +Checkpointing takes time, so a run doesn't checkpoint the moment it reaches a waitpoint. For [`wait.for()`](/wait-for) and [`wait.until()`](/wait-until) it happens 60 seconds into the wait, so anything shorter stays `EXECUTING` and holds its slot for the whole wait. If you're polling in a loop, use an interval comfortably above 60 seconds so the slot is actually released between polls. + ### Waiting for a subtask on a different queue When a parent task triggers and waits for a subtask on a different queue, the parent task will checkpoint and release its concurrency slot once it reaches the wait point. This prevents environment deadlocks where all concurrency slots would be occupied by waiting tasks. diff --git a/docs/snippets/paused-execution-free.mdx b/docs/snippets/paused-execution-free.mdx index 7a9483d451e..6372d1f783b 100644 --- a/docs/snippets/paused-execution-free.mdx +++ b/docs/snippets/paused-execution-free.mdx @@ -1,4 +1,6 @@ In the Trigger.dev Cloud we automatically pause execution of tasks when they are waiting for longer than a few seconds. -When triggering and waiting for subtasks, the parent is checkpointed and while waiting does not count towards compute usage. When waiting for a time period (`wait.for` or `wait.until`), if the wait is longer than 5 seconds we checkpoint and it does not count towards compute usage. \ No newline at end of file +When triggering and waiting for subtasks, the parent is checkpointed and while waiting does not count towards compute usage. When waiting for a time period (`wait.for` or `wait.until`), a wait longer than 5 seconds does not count towards compute usage. + +Free compute isn't the same as freed concurrency: the concurrency slot is only released once we've snapshotted the machine and shut it down. For `wait.for` and `wait.until` that happens 60 seconds into the wait, so anything shorter stays `EXECUTING` and holds its slot for the whole wait. \ No newline at end of file From 1e14e29d7183d6e8badd78a71b45522aae3b6c17 Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Tue, 28 Jul 2026 23:05:20 +0100 Subject: [PATCH 26/67] fix(webapp): restyle the leave and remove team member dialogs (#4411) ## Summary The confirmation dialog for leaving a team or removing a teammate was still built on the old `Alert` primitive: the entire question sat in the title, there was no header divider or `Esc` affordance, and the footer used small buttons pinned to the right. It now uses the standard `Dialog` layout the rest of the dashboard uses. The title is static ("Remove team member" / "Leave team"), the question moves into the body with the person's name and the organization highlighted, and the footer is a bordered row with medium Cancel and confirm buttons. A member who has not set a name is now identified by their email instead of "them". Verified against a local dashboard on both dialogs. Confirming a removal posts the member id, deletes the membership and shows the success toast. Cancel, `Esc`, and Enter while Cancel is focused all close the dialog without issuing a request, leaving the member in place. No release note needed: this is a visual restyle of an existing dialog with no behaviour change. --- .../route.tsx | 80 ++++++++++--------- 1 file changed, 44 insertions(+), 36 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.team/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.team/route.tsx index 3e11636aeec..cc2a95571d0 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.team/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.team/route.tsx @@ -20,16 +20,6 @@ import { UserAvatar } from "~/components/UserProfilePhoto"; import { AdminDebugTooltip } from "~/components/admin/debugTooltip"; import { CopyableText } from "~/components/primitives/CopyableText"; import { PageBody, PageContainer } from "~/components/layout/AppLayout"; -import { - Alert, - AlertCancel, - AlertContent, - AlertDescription, - AlertFooter, - AlertHeader, - AlertTitle, - AlertTrigger, -} from "~/components/primitives/Alert"; import { Button, ButtonContent, LinkButton } from "~/components/primitives/Buttons"; import { PermissionButton } from "~/components/primitives/PermissionButton"; import { DateTime } from "~/components/primitives/DateTime"; @@ -659,8 +649,14 @@ function LeaveRemoveButton({ + Are you sure you want to leave the team? You will no longer have access to{" "} + {organization.title}. To regain access, you + will need to be invited again. + + } actionText="Leave team" /> ); @@ -684,8 +680,16 @@ function LeaveRemoveButton({ + Are you sure you want to remove{" "} + {member.user.name ?? member.user.email} from the + team? They will no longer have access to{" "} + {organization.title}. To regain access, you will + need to invite them again. + + } actionText="Remove from team" /> ); @@ -795,7 +799,7 @@ function LeaveTeamModal({ member: Member; buttonText: string; title: string; - description: string; + description: React.ReactNode; actionText: string; }) { const [open, setOpen] = useState(false); @@ -811,28 +815,32 @@ function LeaveTeamModal({ }); return ( - setOpen(o)}> - + + - - - - {title} - {description} - - - - - -
setOpen(false)}> - - -
- - - + + + {title} +
setOpen(false)}> + + + {description} + + + {actionText} + + } + cancelButton={ + + + + } + /> + +
+ ); } From a11e5ffbc6bb2d2326e1cf7c58d0129eed6ad2bd Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Wed, 29 Jul 2026 10:16:45 +0100 Subject: [PATCH 27/67] fix(webapp): fade overflowing side menu selector labels (#4412) Long organization, project, and environment names in the side menu were cut off mid-character. They now fade out at the right edge like the rest of the side menu items already did. ### Example of faded long names: CleanShot 2026-07-28 at 22 59 24 --- .../app/components/environments/EnvironmentLabel.tsx | 4 +++- .../app/components/navigation/EnvironmentSelector.tsx | 4 ++-- apps/webapp/app/components/navigation/SideMenu.tsx | 10 +++++----- .../webapp/app/components/navigation/SideMenuItem.tsx | 11 ++--------- .../app/components/primitives/labelOverflowFade.ts | 7 +++++++ 5 files changed, 19 insertions(+), 17 deletions(-) create mode 100644 apps/webapp/app/components/primitives/labelOverflowFade.ts diff --git a/apps/webapp/app/components/environments/EnvironmentLabel.tsx b/apps/webapp/app/components/environments/EnvironmentLabel.tsx index 3fd5d526fc1..6f253676e04 100644 --- a/apps/webapp/app/components/environments/EnvironmentLabel.tsx +++ b/apps/webapp/app/components/environments/EnvironmentLabel.tsx @@ -6,6 +6,7 @@ import { } from "~/assets/icons/EnvironmentIcons"; import type { RuntimeEnvironment } from "~/models/runtimeEnvironment.server"; import { cn } from "~/utils/cn"; +import { labelOverflowFadeStyle } from "~/components/primitives/labelOverflowFade"; import { SimpleTooltip } from "~/components/primitives/Tooltip"; import { useEffect, useRef, useState } from "react"; @@ -88,7 +89,7 @@ export function EnvironmentLabel({ tooltipSideOffset?: number; tooltipSide?: "top" | "right" | "bottom" | "left"; disableTooltip?: boolean; - /** When false, the label clips without an ellipsis (side menu fades it in place). Defaults true. */ + /** When false, an overflowing label fades at its right edge instead of ending in an ellipsis. */ truncate?: boolean; }) { const spanRef = useRef(null); @@ -122,6 +123,7 @@ export function EnvironmentLabel({ environmentTextClassName(environment), className )} + style={labelOverflowFadeStyle(!truncate && isTruncated)} > {text} diff --git a/apps/webapp/app/components/navigation/EnvironmentSelector.tsx b/apps/webapp/app/components/navigation/EnvironmentSelector.tsx index 0f51b225457..b2c1d593a0d 100644 --- a/apps/webapp/app/components/navigation/EnvironmentSelector.tsx +++ b/apps/webapp/app/components/navigation/EnvironmentSelector.tsx @@ -94,7 +94,7 @@ export function EnvironmentSelector({ fades in place and scales its width to 0 so it never holds width mid-drag. The selector is also reused outside the side menu (BlankStatePanels, limits) where the var is unset — the 0.2 max-width fallback pins a ~200px cap (0.2 * 1000px) so long names - ellipsis-truncate there instead of widening the control, while opacity stays 1. + fade out there instead of widening the control, while opacity stays 1. */} diff --git a/apps/webapp/app/components/navigation/SideMenu.tsx b/apps/webapp/app/components/navigation/SideMenu.tsx index d5a8172fe67..29eb1fc8520 100644 --- a/apps/webapp/app/components/navigation/SideMenu.tsx +++ b/apps/webapp/app/components/navigation/SideMenu.tsx @@ -1682,9 +1682,9 @@ function OrgSelector({ className="flex min-w-0 items-center gap-1.5 overflow-hidden" style={SIDE_MENU_SELECTOR_LABEL_STYLE} > - + {organization.title} - + - + {project.name ?? "Select a project"} - + - {p.name} + {p.name}
} isSelected={isSelected} diff --git a/apps/webapp/app/components/navigation/SideMenuItem.tsx b/apps/webapp/app/components/navigation/SideMenuItem.tsx index 035fcc54370..d162c064620 100644 --- a/apps/webapp/app/components/navigation/SideMenuItem.tsx +++ b/apps/webapp/app/components/navigation/SideMenuItem.tsx @@ -12,12 +12,10 @@ import { motion } from "framer-motion"; import { usePathName } from "~/hooks/usePathName"; import { cn } from "~/utils/cn"; import { type RenderIcon, Icon } from "../primitives/Icon"; +import { labelOverflowFadeStyle } from "../primitives/labelOverflowFade"; import { SimpleTooltip } from "../primitives/Tooltip"; import { useActiveFavoriteId } from "./favoritePages"; -/** Right-edge fade shown instead of a hard clip, only while the label actually overflows. */ -const LABEL_OVERFLOW_MASK = "linear-gradient(to right, black calc(100% - 1.5rem), transparent)"; - /** * A menu label that fades out at its right edge when (and only when) the text overflows. Text * that fits renders exactly as before, with no mask. Overflow is re-measured when the element @@ -56,12 +54,7 @@ export function SideMenuLabel({ {children} diff --git a/apps/webapp/app/components/primitives/labelOverflowFade.ts b/apps/webapp/app/components/primitives/labelOverflowFade.ts new file mode 100644 index 00000000000..199d81a0fb7 --- /dev/null +++ b/apps/webapp/app/components/primitives/labelOverflowFade.ts @@ -0,0 +1,7 @@ +const LABEL_OVERFLOW_MASK = "linear-gradient(to right, black calc(100% - 1.5rem), transparent)"; + +export function labelOverflowFadeStyle(isOverflowing: boolean) { + return isOverflowing + ? { maskImage: LABEL_OVERFLOW_MASK, WebkitMaskImage: LABEL_OVERFLOW_MASK } + : undefined; +} From 15e160d767fe205c082c37485152cf3279877feb Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Wed, 29 Jul 2026 11:19:16 +0100 Subject: [PATCH 28/67] chore(ci): cache typecheck work across runs (#4415) --- .github/workflows/typecheck.yml | 10 +++++++++- turbo.json | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml index d8521f3936a..950d5a5e981 100644 --- a/.github/workflows/typecheck.yml +++ b/.github/workflows/typecheck.yml @@ -23,11 +23,19 @@ jobs: version: 10.33.2 - name: ⎔ Setup node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 with: node-version: 24.18.0 cache: "pnpm" + - name: Restore Turbo cache + uses: WarpBuilds/cache@40f3443ae7b70e568d6e2070ea897f3df94d7553 # v1 + with: + path: node_modules/.cache/turbo + key: turbo-typecheck-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}-${{ github.sha }} + restore-keys: | + turbo-typecheck-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}- + - name: 📥 Download deps run: pnpm install --frozen-lockfile diff --git a/turbo.json b/turbo.json index c820d115376..bd6f8aca761 100644 --- a/turbo.json +++ b/turbo.json @@ -37,7 +37,8 @@ "cache": false }, "generate": { - "dependsOn": ["^generate"] + "dependsOn": ["^generate"], + "cache": false }, "docker:build": { "outputs": [], From 8d321f8d6e87b3adab73b5442f158308b6c6333b Mon Sep 17 00:00:00 2001 From: DKP <8297864+D-K-P@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:09:45 +0100 Subject: [PATCH 29/67] docs: give docs pages unique title tags and redirect stale pages (#4416) ## Summary Several docs pages rendered identical `` tags, which weakens search indexing and makes results ambiguous. Each affected page now has a unique, descriptive title while keeping its existing sidebar label unchanged. Alongside the retitles: - Removed two stale build-system upgrade pages that were no longer in the navigation, with redirects to the current package upgrade guide. - Redirected the build-extensions group index to its overview page so the two URLs stop sharing a title. - Dropped a leftover orphaned API reference page (its old URL already redirects to the management overview). No links break: nothing in the docs points at the removed pages, and every redirect target exists. --- docs/ai-chat/changelog.mdx | 2 +- docs/ai-chat/how-it-works.mdx | 2 +- docs/bulk-actions.mdx | 3 +- docs/config/extensions/lightpanda.mdx | 2 +- docs/config/extensions/puppeteer.mdx | 2 +- docs/docs.json | 15 + docs/guides/examples/lightpanda.mdx | 2 +- docs/guides/examples/puppeteer.mdx | 2 +- docs/guides/new-build-system-preview.mdx | 438 ----------------------- docs/how-it-works.mdx | 2 +- docs/management/projects/runs.mdx | 4 - docs/runs/bulk-actions.mdx | 3 +- docs/upgrading-beta.mdx | 412 --------------------- 13 files changed, 26 insertions(+), 863 deletions(-) delete mode 100644 docs/guides/new-build-system-preview.mdx delete mode 100644 docs/management/projects/runs.mdx delete mode 100644 docs/upgrading-beta.mdx diff --git a/docs/ai-chat/changelog.mdx b/docs/ai-chat/changelog.mdx index 22f496b91bf..e7567b422b7 100644 --- a/docs/ai-chat/changelog.mdx +++ b/docs/ai-chat/changelog.mdx @@ -1,5 +1,5 @@ --- -title: "Changelog" +title: "AI Agents SDK changelog" sidebarTitle: "Changelog" description: "Changelog for the AI Agents SDK." --- diff --git a/docs/ai-chat/how-it-works.mdx b/docs/ai-chat/how-it-works.mdx index 2003bce8fd9..05308c50ee4 100644 --- a/docs/ai-chat/how-it-works.mdx +++ b/docs/ai-chat/how-it-works.mdx @@ -1,5 +1,5 @@ --- -title: "How it works" +title: "How the AI Agents SDK works" sidebarTitle: "How it works" description: "End-to-end mechanics of a chat.agent turn: the two durable channels per session, the long-lived task that reads and writes them, and how a chat survives refreshes, deploys, and idle gaps." --- diff --git a/docs/bulk-actions.mdx b/docs/bulk-actions.mdx index b40807cfe57..5fa90725077 100644 --- a/docs/bulk-actions.mdx +++ b/docs/bulk-actions.mdx @@ -1,5 +1,6 @@ --- -title: "Bulk actions" +title: "Bulk actions in the dashboard" +sidebarTitle: "Bulk actions" description: "Replay or cancel multiple runs from the dashboard using filters or selected run IDs." --- diff --git a/docs/config/extensions/lightpanda.mdx b/docs/config/extensions/lightpanda.mdx index 1e1efc8f639..7b4a055b124 100644 --- a/docs/config/extensions/lightpanda.mdx +++ b/docs/config/extensions/lightpanda.mdx @@ -1,5 +1,5 @@ --- -title: "Lightpanda" +title: "Lightpanda build extension" sidebarTitle: "lightpanda" description: "Use the lightpanda build extension to add Lightpanda browser to your project" tag: "v4" diff --git a/docs/config/extensions/puppeteer.mdx b/docs/config/extensions/puppeteer.mdx index a2f3091bd7b..4a52cfecad2 100644 --- a/docs/config/extensions/puppeteer.mdx +++ b/docs/config/extensions/puppeteer.mdx @@ -1,5 +1,5 @@ --- -title: "Puppeteer" +title: "Puppeteer build extension" sidebarTitle: "puppeteer" description: "Use the puppeteer build extension to enable support for Puppeteer in your project" --- diff --git a/docs/docs.json b/docs/docs.json index ad5ac311e67..7a3cfa46722 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -721,6 +721,21 @@ ] }, "redirects": [ + { + "source": "/config/extensions", + "destination": "/config/extensions/overview", + "permanent": true + }, + { + "source": "/upgrading-beta", + "destination": "/upgrading-packages", + "permanent": true + }, + { + "source": "/guides/new-build-system-preview", + "destination": "/upgrading-packages", + "permanent": true + }, { "source": "/cli-dev", "destination": "/cli-dev-commands", diff --git a/docs/guides/examples/lightpanda.mdx b/docs/guides/examples/lightpanda.mdx index d2a431624ef..422b1a8fdf9 100644 --- a/docs/guides/examples/lightpanda.mdx +++ b/docs/guides/examples/lightpanda.mdx @@ -1,5 +1,5 @@ --- -title: "Lightpanda" +title: "Lightpanda examples with Trigger.dev" sidebarTitle: "Lightpanda" description: "These examples demonstrate how to use Lightpanda with Trigger.dev." tag: "v4" diff --git a/docs/guides/examples/puppeteer.mdx b/docs/guides/examples/puppeteer.mdx index 9a6f91f018d..1d03814ac4e 100644 --- a/docs/guides/examples/puppeteer.mdx +++ b/docs/guides/examples/puppeteer.mdx @@ -1,5 +1,5 @@ --- -title: "Puppeteer" +title: "Puppeteer examples with Trigger.dev" sidebarTitle: "Puppeteer" description: "These examples demonstrate how to use Puppeteer with Trigger.dev." --- diff --git a/docs/guides/new-build-system-preview.mdx b/docs/guides/new-build-system-preview.mdx deleted file mode 100644 index f4064c2db5f..00000000000 --- a/docs/guides/new-build-system-preview.mdx +++ /dev/null @@ -1,438 +0,0 @@ ---- -title: "Upgrade to new build system" -description: "How to use the new build system preview release" ---- - -Based on feedback and a steady flow of issues from our previous build system, we're happy to share with you a preview release of our new build system that (hopefully) fixes most of the issues you may have faced. - -The main features of the new build sytem are: - -- **Bundling by default**: All dependencies are bundled by default, so you no longer need to specify which dependencies to bundle. This solves a whole bunch of issues related to monorepos. -- **Build extensions**: A new way to extend the build process with custom logic. This is a more flexible and powerful way to extend the build process compared to the old system. (including custom esbuild plugin support) -- **Improved configuration**: We've migrated to using [c12](https://github.com/unjs/c12) to power our configuration system. -- **Improved error handling**: We now do a much better job of reporting of any errors that happen during the indexing process by loading your trigger task files dynamically. -- **Improved cold start times**: Previously, we would load all your trigger task files at once, which could lead to long cold start times. Now we load your trigger task files dynamically, which should improve cold start times. - -## Update packages - -To use the new build system, you have to update to use our preview packages. Update the `@trigger.dev/sdk` package in your package.json: - -```json -"@trigger.dev/sdk": "0.0.0-prerelease-20240911144933" -``` - -You will also need to update your usage of the `trigger.dev` CLI to use the preview release. If you run the CLI via `npx` you can update to the preview release like so: - -```sh -# old way -npx trigger.dev@latest dev - -# using the preview release -npx trigger.dev@0.0.0-prerelease-20240911144933 dev -``` - -If you've added the `trigger.dev` CLI to your `devDependencies`, then you should update the version to point to the preview release: - -```json -"trigger.dev": "0.0.0-prerelease-20240911144933" -``` - -Once you do that make sure you re-install your dependencies using `npm i` or the equivalent with your preferred package manager. - -<Note>If you deploy using GitHub actions, make sure you update the version there too.</Note> - -## Update your `trigger.config.ts` - -The new build system does not effect your trigger task files at all, so those can remain unchanged. However, you may need to make changes to your `trigger.config.ts` file. - -### `defineConfig` - -You should now import the `defineConfig` function from `@trigger.dev/sdk` and export the config as the default export: - -``` -import { defineConfig } from "@trigger.dev/sdk"; - -export default defineConfig({ - project: "<project ref>", -}); -``` - -### Deprecated: `dependenciesToBundle` - -The new build system will bundle all dependencies by default, so `dependenciesToBundle` no longer makes any sense and can be removed. - -#### Externals - -Now that all dependencies are bundled, there are some situations where bundling a dependency doesn't work, and needs to be made external (e.g. when a dependency includes a native module). You can now specify these dependencies as build externals in the `defineConfig` function: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; - -export default defineConfig({ - project: "<project ref>", - build: { - external: ["native-module"], - }, -}); -``` - -`external` is an array of strings, where each string is the name of a dependency that should be made external. Glob expressions are also supported and use the [minimatch](https://github.com/isaacs/minimatch) matcher. - -### `additionalFiles` - -The `additionalFiles` option has been moved to our new build extension system. - -To use build extensions, you'll need to add the `@trigger.dev/build` package to your `devDependencies`: - -```sh -npm add @trigger.dev/build@0.0.0-prerelease-20240911144933 -D -``` - -Now you can import the `additionalFiles` build extension and use it in your `trigger.config.ts` file: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; -import { additionalFiles } from "@trigger.dev/build/extensions/core"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [ - additionalFiles({ files: ["wrangler/wrangler.toml", "./assets/**", "./fonts/**"] }), - ], - }, -}); -``` - -### `additionalPackages` - -The `additionalPackages` option has been moved to our new build extension system. - -To use build extensions, you'll need to add the `@trigger.dev/build` package to your `devDependencies`: - -```sh -npm add @trigger.dev/build@0.0.0-prerelease-20240911144933 -D -``` - -Now you can import the `additionalPackages` build extension and use it in your `trigger.config.ts` file: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; -import { additionalPackages } from "@trigger.dev/build/extensions/core"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [additionalPackages({ packages: ["wrangler"] })], - }, -}); -``` - -### `resolveEnvVars` - -The `resolveEnvVars` export has been moved to our new build extension system. - -To use build extensions, you'll need to add the `@trigger.dev/build` package to your `devDependencies`: - -```sh -npm add @trigger.dev/build@0.0.0-prerelease-20240911144933 -D -``` - -Now you can import the `syncEnvVars` build extension and use it in your `trigger.config.ts` file: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; -import { syncEnvVars } from "@trigger.dev/build/extensions/core"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [ - syncEnvVars(async (params) => { - return { - MY_ENV_VAR: "my-value", - }; - }), - ], - }, -}); -``` - -The `syncEnvVars` callback function works very similarly to the [`resolveEnvVars`](/deploy-environment-variables#sync-env-vars-from-another-service) callback, but now instead of returning an object with a `variables` key that contains the environment variables, you return an object with the environment variables directly (see the example above). - -One other difference is now `params.env` only contains the environment variables that are set in the Trigger.dev environment variables, and not the environment variables from the process. If you want to access the environment variables from the process, you can use `process.env`. - -### emitDecoratorMetadata - -If you make use of decorators in your code, and have enabled the `emitDecoratorMetadata` tsconfig compiler option, you'll need to enable this in the new build sytem using the `emitDecoratorMetadata` build extension: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; -import { emitDecoratorMetadata } from "@trigger.dev/build/extensions/typescript"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [emitDecoratorMetadata()], - }, -}); -``` - -### Prisma - -We've created a build extension to support using Prisma in your Trigger.dev tasks. To use this extension, you'll need to add the `@trigger.dev/build` package to your `devDependencies`: - -```sh -npm add @trigger.dev/build@0.0.0-prerelease-20240911144933 -D -``` - -Then you can import the `prismaExtension` build extension and use it in your `trigger.config.ts` file, passing in the path to your Prisma schema file: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; -import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [ - prismaExtension({ - schema: "prisma/schema.prisma", - }), - ], - }, -}); -``` - -This will make sure that your prisma client is generated during the build process when deploying to Trigger.dev. - -<Note> - This does not have any effect when running the `dev` command, so you'll need to make sure you - generate your client locally first. -</Note> - -If you want to also run migrations during the build process, you can pass in the `migrate` option: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; -import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [ - prismaExtension({ - schema: "prisma/schema.prisma", - migrate: true, - directUrlEnvVarName: "DATABASE_URL_UNPOOLED", // optional - the name of the environment variable that contains the direct database URL if you are using a direct database URL - }), - ], - }, -}); -``` - -If you have multiple `generator` statements defined in your schema file, you can pass in the `clientGenerator` option to specify the `prisma-client-js` generator, which will prevent other generators from being generated: - -<CodeGroup> - -```prisma schema.prisma -datasource db { - provider = "postgresql" - url = env("DATABASE_URL") - directUrl = env("DATABASE_URL_UNPOOLED") -} - -// We only want to generate the prisma-client-js generator -generator client { - provider = "prisma-client-js" -} - -generator kysely { - provider = "prisma-kysely" - output = "../../src/kysely" - enumFileName = "enums.ts" - fileName = "types.ts" -} -``` - -```ts trigger.config.ts -import { defineConfig } from "@trigger.dev/sdk"; -import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [ - prismaExtension({ - schema: "prisma/schema.prisma", - clientGenerator: "client", - }), - ], - }, -}); -``` - -</CodeGroup> - -### audioWaveform - -Previously, we installed [Audio Waveform](https://github.com/bbc/audiowaveform) in the build image. That's been moved to a build extension: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; -import { audioWaveform } from "@trigger.dev/build/extensions/audioWaveform"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [audioWaveform()], // uses verson 1.1.0 of audiowaveform by default - }, -}); -``` - -### esbuild plugins - -You can now add esbuild plugins to customize the build process using the `esbuildPlugin` build extension. The example below shows how to automatically upload sourcemaps to Sentry using their esbuild plugin: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; -import { esbuildPlugin } from "@trigger.dev/build/extensions"; -import { sentryEsbuildPlugin } from "@sentry/esbuild-plugin"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [ - esbuildPlugin( - sentryEsbuildPlugin({ - org: process.env.SENTRY_ORG, - project: process.env.SENTRY_PROJECT, - authToken: process.env.SENTRY_AUTH_TOKEN, - }), - // optional - only runs during the deploy command, and adds the plugin to the end of the list of plugins - { placement: "last", target: "deploy" } - ), - ], - }, -}); -``` - -## Changes to the `trigger.dev` CLI - -### No more typechecking during deploy - -We no longer run typechecking during the deploy command. This was causing issues with some projects, and we found that it wasn't necessary to run typechecking during the deploy command. If you want to run typechecking before deploying to Trigger.dev, you can run the `tsc` command before running the `deploy` command. - -```sh -tsc && npx trigger.dev@0.0.0-prerelease-20240911144933 deploy -``` - -Or if you are using GitHub actions, you can add an additional step to run the `tsc` command before deploying to Trigger.dev. - -```yaml -- name: Install dependencies - run: npm install - -- name: Typecheck - run: npx tsc - -- name: 🚀 Deploy Trigger.dev - env: - TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} - run: | - npx trigger.dev@0.0.0-prerelease-20240911144933 deploy -``` - -### deploy --dry-run - -You can now inspect the build output of your project without actually deploying it to Trigger.dev by using the `--dry-run` flag: - -```sh -npx trigger.dev@0.0.0-prerelease-20240911144933 deploy --dry-run -``` - -This will save the build output and print the path to the build output directory. If you face any issues with deploying, please include the build output in your issue report. - -### --env-file - -You can now pass the path to your local `.env` file using the `--env-file` flag during `dev` and `deploy` commands: - -```sh -npx trigger.dev@0.0.0-prerelease-20240911144933 dev --env-file ../../.env -npx trigger.dev@0.0.0-prerelease-20240911144933 deploy --env-file ../../.env -``` - -The `.env` file works slightly differently in `dev` vs `deploy`: - -- In `dev`, the `.env` file is loaded into the CLI's `process.env` and also into the environment variables of the Trigger.dev environment. -- In `deploy`, the `.env` file is loaded into the CLI's `process.env` but not into the environment variables of the Trigger.dev environment. If you want to sync the environment variables from the `.env` file to the Trigger.dev environment variables, you can use the `syncEnvVars` build extension. - -### dev debugging in VS Code - -Debugging your tasks code in `dev` is now supported via VS Code, without having to pass in any additional flags. Create a launch configuration in `.vscode/launch.json`: - -```json launch.json -{ - "version": "0.2.0", - "configurations": [ - { - "name": "Trigger.dev: Dev", - "type": "node", - "request": "launch", - "cwd": "${workspaceFolder}", - "runtimeExecutable": "npx", - "runtimeArgs": ["trigger.dev@0.0.0-prerelease-20240911144933", "dev"], - "skipFiles": ["<node_internals>/**"], - "sourceMaps": true - } - ] -} -``` - -Then you can start debugging your tasks code by selecting the `Trigger.dev: Dev` configuration in the debug panel, and set breakpoints in your tasks code. - -### TRIGGER_ACCESS_TOKEN in dev - -You can now authenticate the `dev` command using the `TRIGGER_ACCESS_TOKEN` environment variable. Previously this was only supported in the `deploy` command. - -```sh -TRIGGER_ACCESS_TOKEN=<your access token> npx trigger.dev@0.0.0-prerelease-20240911144933 dev -``` - -### Better deploy support for self-hosters - -You can now specify a custom registry and namespace when deploying via a self-hosted instance of Trigger.dev: - -```sh -npx trigger.dev@0.0.0-prerelease-20240911144933 deploy --self-hosted --load-image --push --registry docker.io --namespace mydockerhubusername -``` - -All you have to do is create a repository in dockerhub that matches the project ref of your Trigger.dev project (e.g. `proj_rrkpdguyagvsoktglnod`) - -<Note> - Docker Hub will automatically create a repository the first time you push, which is public by - default. If you want to keep these images private, make sure you create the repository before you - first run the `deploy` command -</Note> - -## Known issues - -- Path aliases are not yet support in your `trigger.config.ts` file. To workaround this issue you'll need to rewrite path aliases to their relative paths. (See [this](https://github.com/unjs/jiti/issues/166) and [this](https://knip.dev/reference/known-issues#path-aliases-in-config-files)) for more info. -- `*.test.ts` and `.spec.ts` files inside the trigger dirs will be bundled and could cause issues. You'll need to move these files outside of the trigger dirs to avoid this issue. - -## Changelog - -### Changes and fixes in `0.0.0-prerelease-20240911144933` - -- Fixed an issue where empty env vars in dev runs were overriding local `.env` file values. -- Fixed an issue when importing v2 `@trigger.dev/sdk` would throw an error because of a missing package.json -- Fixed node10 moduleResolution with some types -- Added the ability to push to a custom registry when deploying to a self-hosted instance of Trigger.dev -- Fix stuck dev runs when a child run fails with a process.exit -- No longer ignoring `--project-ref` in the `deploy` command -- No longer ignoring the `--config` option in the `deploy` command -- Fixed the flushing of logs to the server when deployed. -- `emitDecoratorMetadata` build extension now works when tsconfig.json file extends another tsconfig.json file -- We now have the ability to force certain packages to be external (i.e. not be bundled), starting with `headers-generator` -- You can now call `init` with the `--javascript` flag to initialize a project with JavaScript instead of TypeScript -- Add support for Prisma TypedSQL in the prisma extension diff --git a/docs/how-it-works.mdx b/docs/how-it-works.mdx index c7d3e089c1f..99c08e944d8 100644 --- a/docs/how-it-works.mdx +++ b/docs/how-it-works.mdx @@ -1,5 +1,5 @@ --- -title: "How it works" +title: "How Trigger.dev works" sidebarTitle: "How it works" description: "Understand how Trigger.dev works and how it can help you." --- diff --git a/docs/management/projects/runs.mdx b/docs/management/projects/runs.mdx deleted file mode 100644 index ad5a3046836..00000000000 --- a/docs/management/projects/runs.mdx +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: "List runs" -openapi: "v3-openapi GET /api/v1/projects/{projectRef}/runs" ---- diff --git a/docs/runs/bulk-actions.mdx b/docs/runs/bulk-actions.mdx index 10a7caa90dc..879ab57342c 100644 --- a/docs/runs/bulk-actions.mdx +++ b/docs/runs/bulk-actions.mdx @@ -1,5 +1,6 @@ --- -title: "Bulk actions" +title: "Bulk actions with the SDK" +sidebarTitle: "Bulk actions" description: "Cancel or replay many runs from the SDK using run IDs or SDK run-list filters." --- diff --git a/docs/upgrading-beta.mdx b/docs/upgrading-beta.mdx deleted file mode 100644 index 532526fc4d8..00000000000 --- a/docs/upgrading-beta.mdx +++ /dev/null @@ -1,412 +0,0 @@ ---- -title: "Upgrade to new build system" -sidebarTitle: "Beta upgrade" -description: "How to update to 3.0.0 from the beta" ---- - -import DebuggingInVSCode from '/snippets/debugging_in_vscode.mdx'; - -The Trigger.dev packages are now at version `3.0.x` in the `latest` tag. This is our first official release of v3 under the latest tag, and we recommend anyone still using packages in the `beta` tag to upgrade to the latest version. This guide will help you upgrade your project to the latest version of Trigger.dev. - -The major changes in this release are a new build system, which is more flexible and powerful than the previous build system. We've also made some changes to the `trigger.dev` CLI to improve the developer experience. - -The main features of the new build sytem are: - -- **Bundling by default**: All dependencies are bundled by default, so you no longer need to specify which dependencies to bundle. This solves a whole bunch of issues related to monorepos. -- **Build extensions**: A new way to extend the build process with custom logic. This is a more flexible and powerful way to extend the build process compared to the old system. (including custom esbuild plugin support) -- **Improved configuration**: We've migrated to using [c12](https://github.com/unjs/c12) to power our configuration system. -- **Improved error handling**: We now do a much better job of reporting of any errors that happen during the indexing process by loading your trigger task files dynamically. -- **Improved cold start times**: Previously, we would load all your trigger task files at once, which could lead to long cold start times. Now we load your trigger task files dynamically, which should improve cold start times. - -## Update packages - -To use the new build system, you have to update to use our latest packages. Update the `@trigger.dev/sdk` package in your package.json: - -```json -"@trigger.dev/sdk": "^3.0.0", -``` - -You will also need to update your usage of the `trigger.dev` CLI to use the latest release. If you run the CLI via `npx` you can update to the latest release like so: - -```sh -# old way -npx trigger.dev@3.0.0-beta.56 dev - -# using the latest release -npx trigger.dev@latest dev -``` - -If you've added the `trigger.dev` CLI to your `devDependencies`, then you should update the version to point to the latest release: - -```json -"trigger.dev": "^3.0.0", -``` - -Once you do that make sure you re-install your dependencies using `npm i` or the equivalent with your preferred package manager. - -<Note>If you deploy using GitHub actions, make sure you update the version there too.</Note> - -## Update your `trigger.config.ts` - -The new build system does not effect your trigger task files at all, so those can remain unchanged. However, you may need to make changes to your `trigger.config.ts` file. - -### `defineConfig` - -You should now import the `defineConfig` function from `@trigger.dev/sdk` and export the config as the default export: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; - -export default defineConfig({ - project: "<project ref>", -}); -``` - -### Deprecated: `dependenciesToBundle` - -The new build system will bundle all dependencies by default, so `dependenciesToBundle` no longer makes any sense and can be removed. - -#### Externals - -Now that all dependencies are bundled, there are some situations where bundling a dependency doesn't work, and needs to be made external (e.g. when a dependency includes a native module). You can now specify these dependencies as build externals in the `defineConfig` function: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; - -export default defineConfig({ - project: "<project ref>", - build: { - external: ["native-module"], - }, -}); -``` - -`external` is an array of strings, where each string is the name of a dependency that should be made external. Glob expressions are also supported and use the [minimatch](https://github.com/isaacs/minimatch) matcher. - -### additionalFiles - -The `additionalFiles` option has been moved to our new build extension system. - -To use build extensions, you'll need to add the `@trigger.dev/build` package to your `devDependencies`: - -```sh -npm add @trigger.dev/build@latest -D -``` - -Now you can import the `additionalFiles` build extension and use it in your `trigger.config.ts` file: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; -import { additionalFiles } from "@trigger.dev/build/extensions/core"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [ - additionalFiles({ files: ["wrangler/wrangler.toml", "./assets/**", "./fonts/**"] }), - ], - }, -}); -``` - -### additionalPackages - -The `additionalPackages` option has been moved to our new build extension system. - -To use build extensions, you'll need to add the `@trigger.dev/build` package to your `devDependencies`: - -```sh -npm add @trigger.dev/build@latest -D -``` - -Now you can import the `additionalPackages` build extension and use it in your `trigger.config.ts` file: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; -import { additionalPackages } from "@trigger.dev/build/extensions/core"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [additionalPackages({ packages: ["wrangler"] })], - }, -}); -``` - -### resolveEnvVars - -The `resolveEnvVars` export has been moved to our new build extension system. - -To use build extensions, you'll need to add the `@trigger.dev/build` package to your `devDependencies`: - -```sh -npm add @trigger.dev/build@latest -D -``` - -Now you can import the `syncEnvVars` build extension and use it in your `trigger.config.ts` file: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; -import { syncEnvVars } from "@trigger.dev/build/extensions/core"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [ - syncEnvVars(async (params) => { - return { - MY_ENV_VAR: "my-value", - }; - }), - ], - }, -}); -``` - -The `syncEnvVars` callback function works very similarly to the deprecated `resolveEnvVars` handler, but now instead of returning an object with a `variables` key that contains the environment variables, you return an object with the environment variables directly (see the example above). - -One other difference is now `params.env` only contains the environment variables that are set in the Trigger.dev environment variables, and not the environment variables from the process. If you want to access the environment variables from the process, you can use `process.env`. - -See the [syncEnvVars](/deploy-environment-variables#sync-env-vars-from-another-service) documentation for more information. - -### emitDecoratorMetadata - -If you make use of decorators in your code, and have enabled the `emitDecoratorMetadata` tsconfig compiler option, you'll need to enable this in the new build sytem using the `emitDecoratorMetadata` build extension: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; -import { emitDecoratorMetadata } from "@trigger.dev/build/extensions/typescript"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [emitDecoratorMetadata()], - }, -}); -``` - -### Prisma - -We've created a build extension to support using Prisma in your Trigger.dev tasks. To use this extension, you'll need to add the `@trigger.dev/build` package to your `devDependencies`: - -```sh -npm add @trigger.dev/build@latest -D -``` - -Then you can import the `prismaExtension` build extension and use it in your `trigger.config.ts` file, passing in the path to your Prisma schema file: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; -import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [ - prismaExtension({ - schema: "prisma/schema.prisma", - }), - ], - }, -}); -``` - -This will make sure that your prisma client is generated during the build process when deploying to Trigger.dev. - -<Note> - This does not have any effect when running the `dev` command, so you'll need to make sure you - generate your client locally first. -</Note> - -If you want to also run migrations during the build process, you can pass in the `migrate` option: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; -import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [ - prismaExtension({ - schema: "prisma/schema.prisma", - migrate: true, - directUrlEnvVarName: "DATABASE_URL_UNPOOLED", // optional - the name of the environment variable that contains the direct database URL if you are using a direct database URL - }), - ], - }, -}); -``` - -If you have multiple `generator` statements defined in your schema file, you can pass in the `clientGenerator` option to specify the `prisma-client-js` generator, which will prevent other generators from being generated: - -<CodeGroup> - -```prisma schema.prisma -datasource db { - provider = "postgresql" - url = env("DATABASE_URL") - directUrl = env("DATABASE_URL_UNPOOLED") -} - -// We only want to generate the prisma-client-js generator -generator client { - provider = "prisma-client-js" -} - -generator kysely { - provider = "prisma-kysely" - output = "../../src/kysely" - enumFileName = "enums.ts" - fileName = "types.ts" -} -``` - -```ts trigger.config.ts -import { defineConfig } from "@trigger.dev/sdk"; -import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [ - prismaExtension({ - schema: "prisma/schema.prisma", - clientGenerator: "client", - }), - ], - }, -}); -``` - -</CodeGroup> - -### audioWaveform - -Previously, we installed [Audio Waveform](https://github.com/bbc/audiowaveform) in the build image. That's been moved to a build extension: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; -import { audioWaveform } from "@trigger.dev/build/extensions/audioWaveform"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [audioWaveform()], // uses verson 1.1.0 of audiowaveform by default - }, -}); -``` - -### esbuild plugins - -You can now add esbuild plugins to customize the build process using the `esbuildPlugin` build extension. The example below shows how to automatically upload sourcemaps to Sentry using their esbuild plugin: - -```ts -import { defineConfig } from "@trigger.dev/sdk"; -import { esbuildPlugin } from "@trigger.dev/build/extensions"; -import { sentryEsbuildPlugin } from "@sentry/esbuild-plugin"; - -export default defineConfig({ - project: "<project ref>", - build: { - extensions: [ - esbuildPlugin( - sentryEsbuildPlugin({ - org: process.env.SENTRY_ORG, - project: process.env.SENTRY_PROJECT, - authToken: process.env.SENTRY_AUTH_TOKEN, - }), - // optional - only runs during the deploy command, and adds the plugin to the end of the list of plugins - { placement: "last", target: "deploy" } - ), - ], - }, -}); -``` - -## Changes to the `trigger.dev` CLI - -### No more typechecking during deploy - -We no longer run typechecking during the deploy command. This was causing issues with some projects, and we found that it wasn't necessary to run typechecking during the deploy command. If you want to run typechecking before deploying to Trigger.dev, you can run the `tsc` command before running the `deploy` command. - -```sh -tsc && npx trigger.dev@latest deploy -``` - -Or if you are using GitHub actions, you can add an additional step to run the `tsc` command before deploying to Trigger.dev. - -```yaml -- name: Install dependencies - run: npm install - -- name: Typecheck - run: npx tsc - -- name: 🚀 Deploy Trigger.dev - env: - TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} - run: | - npx trigger.dev@latest deploy -``` - -### deploy `--dry-run` - -You can now inspect the build output of your project without actually deploying it to Trigger.dev by using the `--dry-run` flag: - -```sh -npx trigger.dev@latest deploy --dry-run -``` - -This will save the build output and print the path to the build output directory. If you face any issues with deploying, please include the build output in your issue report. - -### `--env-file` - -You can now pass the path to your local `.env` file using the `--env-file` flag during `dev` and `deploy` commands: - -```sh -npx trigger.dev@latest dev --env-file ../../.env -npx trigger.dev@latest deploy --env-file ../../.env -``` - -The `.env` file works slightly differently in `dev` vs `deploy`: - -- In `dev`, the `.env` file is loaded into the CLI's `process.env` and also into the environment variables of the Trigger.dev environment. -- In `deploy`, the `.env` file is loaded into the CLI's `process.env` but not into the environment variables of the Trigger.dev environment. If you want to sync the environment variables from the `.env` file to the Trigger.dev environment variables, you can use the `syncEnvVars` build extension. - -### dev debugging in VS Code - -<DebuggingInVSCode /> - -### TRIGGER_ACCESS_TOKEN in dev - -You can now authenticate the `dev` command using the `TRIGGER_ACCESS_TOKEN` environment variable. Previously this was only supported in the `deploy` command. - -```sh -TRIGGER_ACCESS_TOKEN=<your access token> npx trigger.dev@latest dev -``` - -### Better deploy support for self-hosters - -You can now specify a custom registry and namespace when deploying via a self-hosted instance of Trigger.dev: - -```sh -npx trigger.dev@latest deploy \ - --self-hosted \ - --load-image \ - --registry docker.io \ - --namespace mydockerhubusername -``` - -All you have to do is create a repository in dockerhub that matches the project ref of your Trigger.dev project (e.g. `proj_rrkpdguyagvsoktglnod`) - -<Note> - Docker Hub will automatically create a repository the first time you push, which is public by - default. If you want to keep these images private, make sure you create the repository before you - first run the `deploy` command -</Note> - -## Known issues - -- Path aliases are not yet support in your `trigger.config.ts` file. To workaround this issue you'll need to rewrite path aliases to their relative paths. (See [this](https://github.com/unjs/jiti/issues/166) and [this](https://knip.dev/reference/known-issues#path-aliases-in-config-files)) for more info. -- `*.test.ts` and `.spec.ts` files inside the trigger dirs will be bundled and could cause issues. You'll need to move these files outside of the trigger dirs to avoid this issue. From 639eaf6e824040aa0d1f8b229acd3882f3a879ac Mon Sep 17 00:00:00 2001 From: "claude[bot]" <209825114+claude[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:31:59 +0100 Subject: [PATCH 30/67] fix(webapp): don't apply an invite's role to an existing org member (#4409) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit <!-- ccr-slack-attribution --> _Requested via [Slack thread](https://triggerdotdev.slack.com/archives/C097ZHVKZFA/p1785249693523749)_ ## Summary Accepting an old invitation could change the role of someone who was already in the organization. A long-pending invite can carry a lower role than the member has since been promoted to, so accepting it was a silent demotion. When the accepting user was the organization's only Owner, the role layer refused that demotion, and the refusal (an expected, protective outcome) was logged as an error. An invitation now only sets a role on a membership the accept actually created, and people who are already in an organization are skipped when invitations are sent. ## How `acceptInvite` already skipped the `OrgMember` create when it found an existing membership, but the `rbac.setUserRole` call below it was gated only on `invite.rbacRoleId`. It now also tracks whether this accept created the membership. A create that loses the unique-constraint race counts as pre-existing, since whichever flow won it owns that membership's role. Skipping existing members outright would regress one case: a member with no RBAC role at all would never receive the invitation's role. `ensureOrgMember` handles that with `healMissingRoleAssignment`, which fills in a null role but never overwrites a real one, so `assignInviteRbacRole` takes the same gate. An established role is never touched; an absent one is filled in. `assignInviteRbacRole` branches on the result's machine-readable `code` instead of logging every refusal at `error`. `last_owner` goes to `logger.info`, matching the two directory-sync role paths; everything else, including a refusal that carries no code, goes to `logger.warn`. The helper is best-effort and never throws, so no outcome it produces warrants `error`. No string matching on the error text is involved. `inviteMembers` resolves the organization's members by email and skips those addresses before creating invites. The invite table's `@@unique([organizationId, email])` only dedupes *pending invites*, so it could never catch this. ## Invite surfaces Skipping addresses means a batch can now come back empty, and neither caller handled that: - The dashboard action built its redirect from `invites[0].organization`, so a batch where every address was skipped threw a `TypeError` that reached the admin as a raw error string. It also reported the submitted count rather than the created one. It now names what it skipped ("No invitations sent: 1 already a member of this organization") and counts what it actually created. - The invites API derived `alreadyInvited` as "everything not created", so an existing member was reported as though they had already been invited. `inviteMembers` now returns the two groups separately and the endpoint reports `alreadyMembers` alongside `alreadyInvited`. ## Testing `apps/webapp/test/member.server.test.ts` passes 16/16 locally, up from 12. Getting there needed a harness fix. The `~/db.server` mock did not export `Prisma`, so any code reaching `PrismaNamespace.PrismaClientKnownRequestError` threw before it could branch, leaving every duplicate-key path in `member.server.ts` unreachable from tests. The mock now re-exports the real `Prisma`, and there is a case covering the pending-invite skip. New cases: the invite role is applied when the accept creates the membership; it is not applied when the member already has a role; it is applied when an existing member has no role assigned; the organization is still joined when the assignment is refused with `last_owner`; and `inviteMembers` reports members separately from pending invites. Forcing the gate off fails exactly the "already has a role" case, so the coverage is load-bearing. `pnpm run typecheck --filter webapp` and `oxfmt --check` both pass. ## Changelog Accepting an old invitation could change the role of someone who was already in the organization. An invitation now leaves an existing member's role untouched, people who are already in an organization are no longer sent invitations to it, and the invite form says which addresses it skipped instead of failing with an unhelpful error. --- ## ✅ Checklist - [x] I have followed every step in the [contributing guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md) - [x] The PR title follows the convention. - [x] I ran and tested the code works. ## Screenshots No visual changes. The invite form's toast copy changes, as described above. --------- Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Matt Aitken <matt@mattaitken.com> --- ...invite-role-change-for-existing-members.md | 6 + apps/webapp/app/models/member.server.ts | 61 +++- .../route.tsx | 35 ++- .../routes/api.v1.orgs.$orgParam.invites.ts | 8 +- apps/webapp/test/member.server.test.ts | 263 ++++++++++++++++-- 5 files changed, 333 insertions(+), 40 deletions(-) create mode 100644 .server-changes/fix-invite-role-change-for-existing-members.md diff --git a/.server-changes/fix-invite-role-change-for-existing-members.md b/.server-changes/fix-invite-role-change-for-existing-members.md new file mode 100644 index 00000000000..ad4fb18b622 --- /dev/null +++ b/.server-changes/fix-invite-role-change-for-existing-members.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Accepting an old invitation could change the role of someone who was already in the organization. An invitation now leaves an existing member's role untouched, people who are already in an organization are no longer sent invitations to it, and the invite form now says which addresses it skipped instead of failing with an unhelpful error. diff --git a/apps/webapp/app/models/member.server.ts b/apps/webapp/app/models/member.server.ts index b786ce7e352..4167738413b 100644 --- a/apps/webapp/app/models/member.server.ts +++ b/apps/webapp/app/models/member.server.ts @@ -100,6 +100,17 @@ export async function inviteMembers({ throw new Error("User does not have access to this organization"); } + const uniqueEmails = new Set(emails); + + const existingMembers = await prisma.orgMember.findMany({ + where: { + organizationId: org.id, + user: { email: { in: [...uniqueEmails] } }, + }, + select: { user: { select: { email: true } } }, + }); + const existingMemberEmails = new Set(existingMembers.map((member) => member.user.email)); + // Create one invite per unique email and return ONLY the invites actually // created by this call. A P2002 means the email is already invited to this org // (unique org+email) — skip it so one duplicate can't fail the batch, and @@ -108,8 +119,15 @@ export async function inviteMembers({ const created: Prisma.OrgMemberInviteGetPayload<{ include: { organization: true; inviter: true }; }>[] = []; + const alreadyMembers: string[] = []; + const alreadyInvited: string[] = []; + + for (const email of uniqueEmails) { + if (existingMemberEmails.has(email)) { + alreadyMembers.push(email); + continue; + } - for (const email of new Set(emails)) { try { const invite = await prisma.orgMemberInvite.create({ data: { @@ -131,13 +149,14 @@ export async function inviteMembers({ error instanceof PrismaNamespace.PrismaClientKnownRequestError && error.code === "P2002" ) { + alreadyInvited.push(email); continue; } throw error; } } - return created; + return { created, alreadyMembers, alreadyInvited }; } export async function getInviteFromToken({ token }: { token: string }) { @@ -264,24 +283,41 @@ async function assignInviteRbacRole({ userId, organizationId, rbacRoleId, + onlyWhenUnassigned, }: { userId: string; organizationId: string; rbacRoleId: string; + onlyWhenUnassigned: boolean; }) { try { + if (onlyWhenUnassigned) { + const currentRole = await rbac.getUserRole({ userId, organizationId }); + if (currentRole !== null) { + return; + } + } + const roleResult = await rbac.setUserRole({ userId, organizationId, roleId: rbacRoleId, }); if (!roleResult.ok) { - logger.error("acceptInvite: skipped RBAC role assignment", { - organizationId, - userId, - rbacRoleId, - reason: roleResult.error, - }); + if (roleResult.code === "last_owner") { + logger.info("acceptInvite: kept last Owner, skipped RBAC role assignment", { + organizationId, + userId, + rbacRoleId, + }); + } else { + logger.warn("acceptInvite: skipped RBAC role assignment", { + organizationId, + userId, + rbacRoleId, + reason: roleResult.error, + }); + } } } catch (error) { logger.error("acceptInvite: RBAC role assignment threw", { @@ -415,6 +451,8 @@ export async function acceptInvite({ }, }); + let membershipCreated = false; + if (!member) { try { member = await prisma.orgMember.create({ @@ -424,6 +462,7 @@ export async function acceptInvite({ role: invite.role, }, }); + membershipCreated = true; } catch (error) { if ( error instanceof PrismaNamespace.PrismaClientKnownRequestError && @@ -473,16 +512,12 @@ export async function acceptInvite({ const remainingInvites = await getUsersInvites({ email: user.email }); - // If the invite carried an explicit RBAC role, assign it. Best-effort: the - // invite is already consumed and membership created above, so a failure here - // — a returned {ok:false} or a thrown error from the plugin — must not block - // joining the org. Swallow and log either way; without the catch a plugin - // throw escapes and turns the whole invite-accept into a 400. if (invite.rbacRoleId) { await assignInviteRbacRole({ userId: user.id, organizationId: invite.organization.id, rbacRoleId: invite.rbacRoleId, + onlyWhenUnassigned: !membershipCreated, }); } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.invite/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.invite/route.tsx index 6a5a0e7a010..b6fe0b09c68 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.invite/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.invite/route.tsx @@ -28,7 +28,7 @@ import { $replica } from "~/db.server"; import { env } from "~/env.server"; import { useOrganization } from "~/hooks/useOrganizations"; import { inviteMembers } from "~/models/member.server"; -import { redirectWithSuccessMessage } from "~/models/message.server"; +import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; import { resolveOrgIdFromSlug } from "~/models/organization.server"; import { TeamPresenter } from "~/presenters/TeamPresenter.server"; import { scheduleEmail } from "~/services/scheduleEmail.server"; @@ -127,6 +127,20 @@ const schema = z.object({ rbacRoleId: z.string().optional(), }); +function describeSkippedInvites(alreadyMembers: string[], alreadyInvited: string[]) { + const parts: string[] = []; + + if (alreadyMembers.length > 0) { + parts.push(simplur`${alreadyMembers.length} already [a member|members] of this organization`); + } + + if (alreadyInvited.length > 0) { + parts.push(simplur`${alreadyInvited.length} already invited`); + } + + return parts.join(" and "); +} + export const action = dashboardAction( { params: Params, @@ -201,7 +215,11 @@ export const action = dashboardAction( } try { - const invites = await inviteMembers({ + const { + created: invites, + alreadyMembers, + alreadyInvited, + } = await inviteMembers({ slug: organizationSlug, emails: submission.value.emails, userId, @@ -224,10 +242,19 @@ export const action = dashboardAction( } } + const teamPath = organizationTeamPath({ slug: organizationSlug }); + const skipped = describeSkippedInvites(alreadyMembers, alreadyInvited); + + if (invites.length === 0) { + return redirectWithErrorMessage(teamPath, request, `No invitations sent: ${skipped}.`); + } + return redirectWithSuccessMessage( - organizationTeamPath(invites[0].organization), + teamPath, request, - simplur`${submission.value.emails.length} member[|s] invited` + skipped + ? simplur`${invites.length} member[|s] invited. Skipped ${skipped}.` + : simplur`${invites.length} member[|s] invited` ); } catch (error: any) { return json({ errors: { body: error.message } }, { status: 400 }); diff --git a/apps/webapp/app/routes/api.v1.orgs.$orgParam.invites.ts b/apps/webapp/app/routes/api.v1.orgs.$orgParam.invites.ts index 76aeeb171b7..36e56e060eb 100644 --- a/apps/webapp/app/routes/api.v1.orgs.$orgParam.invites.ts +++ b/apps/webapp/app/routes/api.v1.orgs.$orgParam.invites.ts @@ -57,9 +57,7 @@ export const action = createActionPATApiRoute( return json({ error: "Membership is managed by Directory Sync" }, { status: 403 }); } - // Returns only the invites created by this call; already-invited emails are - // skipped (re-sending is the dashboard's dedicated resend flow, not this). - const created = await inviteMembers({ + const { created, alreadyMembers, alreadyInvited } = await inviteMembers({ slug: organization.slug, emails: body.emails, userId: authentication.userId, @@ -85,13 +83,11 @@ export const action = createActionPATApiRoute( // Report per-email outcome so callers aren't misled by an empty list on // re-invite. 201 when something was created, 200 when everything already // existed. - const createdEmails = new Set(created.map((invite) => invite.email)); - const alreadyInvited = [...new Set(body.emails)].filter((email) => !createdEmails.has(email)); - return json( { invited: created.map((invite) => ({ id: invite.id, email: invite.email })), alreadyInvited, + alreadyMembers, }, { status: created.length > 0 ? 201 : 200 } ); diff --git a/apps/webapp/test/member.server.test.ts b/apps/webapp/test/member.server.test.ts index 8d4a9128642..722c49aff18 100644 --- a/apps/webapp/test/member.server.test.ts +++ b/apps/webapp/test/member.server.test.ts @@ -1,31 +1,60 @@ import { randomBytes } from "node:crypto"; -import { describe, expect, vi } from "vitest"; +import { beforeEach, describe, expect, vi } from "vitest"; import type { PrismaClient } from "@trigger.dev/database"; const prismaHolder = vi.hoisted(() => ({ client: null as PrismaClient | null, })); +type SetUserRoleParams = { userId: string; organizationId: string; roleId: string }; +type SetUserRoleResult = { ok: true } | { ok: false; error: string; code?: "last_owner" }; +type CurrentRole = { id: string } | null; + +const rbacHolder = vi.hoisted(() => ({ + setUserRoleCalls: [] as SetUserRoleParams[], + setUserRoleResult: { ok: true } as SetUserRoleResult, + currentRole: null as CurrentRole, +})); + vi.mock("~/services/rbac.server", () => ({ rbac: { - setUserRole: async () => ({ ok: true as const }), + getUserRole: async () => rbacHolder.currentRole, + setUserRole: async (params: SetUserRoleParams) => { + rbacHolder.setUserRoleCalls.push(params); + return rbacHolder.setUserRoleResult; + }, }, })); -vi.mock("~/db.server", () => ({ - get prisma() { - if (!prismaHolder.client) { - throw new Error("test prisma not set"); - } - return prismaHolder.client; - }, - get $replica() { - if (!prismaHolder.client) { - throw new Error("test prisma not set"); - } - return prismaHolder.client; - }, -})); +function resetRbac(result: SetUserRoleResult = { ok: true }, currentRole: CurrentRole = null) { + rbacHolder.setUserRoleCalls.length = 0; + rbacHolder.setUserRoleResult = result; + rbacHolder.currentRole = currentRole; +} + +beforeEach(() => { + resetRbac(); +}); + +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + + return { + Prisma, + get prisma() { + if (!prismaHolder.client) { + throw new Error("test prisma not set"); + } + return prismaHolder.client; + }, + get $replica() { + if (!prismaHolder.client) { + throw new Error("test prisma not set"); + } + return prismaHolder.client; + }, + }; +}); import { postgresTest } from "@internal/testcontainers"; @@ -39,7 +68,7 @@ function randomHex(len = 12): string { async function seedInviteFixture( prisma: PrismaClient, - opts: { activeProjectCount: number; deletedProjectCount?: number } + opts: { activeProjectCount: number; deletedProjectCount?: number; rbacRoleId?: string } ) { const suffix = randomHex(8); const inviter = await prisma.user.create({ @@ -99,6 +128,7 @@ async function seedInviteFixture( organizationId: organization.id, inviterId: inviter.id, role: "MEMBER", + rbacRoleId: opts.rbacRoleId ?? null, }, }); @@ -253,6 +283,205 @@ describe("acceptInvite", () => { expect(member).toBeNull(); } ); + + postgresTest( + "applies the invite RBAC role when it creates the membership", + { timeout: 60_000 }, + async ({ prisma }) => { + prismaHolder.client = prisma; + const { acceptInvite } = await import("../app/models/member.server"); + + const { invitee, organization, invite } = await seedInviteFixture(prisma, { + activeProjectCount: 1, + rbacRoleId: "role_admin", + }); + + await acceptInvite({ + inviteId: invite.id, + organizationId: organization.id, + user: { id: invitee.id, email: invitee.email }, + }); + + expect(rbacHolder.setUserRoleCalls).toEqual([ + { userId: invitee.id, organizationId: organization.id, roleId: "role_admin" }, + ]); + } + ); + + postgresTest( + "does not apply the invite RBAC role to a member who already has a role", + { timeout: 60_000 }, + async ({ prisma }) => { + prismaHolder.client = prisma; + resetRbac({ ok: true }, { id: "role_owner" }); + const { acceptInvite } = await import("../app/models/member.server"); + + const { invitee, organization, invite } = await seedInviteFixture(prisma, { + activeProjectCount: 1, + rbacRoleId: "role_admin", + }); + + await prisma.orgMember.create({ + data: { + organizationId: organization.id, + userId: invitee.id, + role: "MEMBER", + }, + }); + + await acceptInvite({ + inviteId: invite.id, + organizationId: organization.id, + user: { id: invitee.id, email: invitee.email }, + }); + + expect(rbacHolder.setUserRoleCalls).toEqual([]); + + const remainingInvite = await prisma.orgMemberInvite.findFirst({ + where: { id: invite.id }, + }); + expect(remainingInvite).toBeNull(); + + const member = await prisma.orgMember.findFirst({ + where: { userId: invitee.id, organizationId: organization.id }, + }); + expect(member).not.toBeNull(); + } + ); + + postgresTest( + "applies the invite RBAC role to an existing member who has no role assigned", + { timeout: 60_000 }, + async ({ prisma }) => { + prismaHolder.client = prisma; + resetRbac({ ok: true }, null); + const { acceptInvite } = await import("../app/models/member.server"); + + const { invitee, organization, invite } = await seedInviteFixture(prisma, { + activeProjectCount: 1, + rbacRoleId: "role_admin", + }); + + await prisma.orgMember.create({ + data: { + organizationId: organization.id, + userId: invitee.id, + role: "MEMBER", + }, + }); + + await acceptInvite({ + inviteId: invite.id, + organizationId: organization.id, + user: { id: invitee.id, email: invitee.email }, + }); + + expect(rbacHolder.setUserRoleCalls).toEqual([ + { userId: invitee.id, organizationId: organization.id, roleId: "role_admin" }, + ]); + } + ); + + postgresTest( + "still joins the org when the RBAC role assignment is refused", + { timeout: 60_000 }, + async ({ prisma }) => { + prismaHolder.client = prisma; + resetRbac({ + ok: false, + error: "An organisation must have at least one Owner", + code: "last_owner", + }); + const { acceptInvite } = await import("../app/models/member.server"); + + const { invitee, organization, invite } = await seedInviteFixture(prisma, { + activeProjectCount: 1, + rbacRoleId: "role_admin", + }); + + const { organization: joinedOrg } = await acceptInvite({ + inviteId: invite.id, + organizationId: organization.id, + user: { id: invitee.id, email: invitee.email }, + }); + + expect(joinedOrg.id).toBe(organization.id); + expect(rbacHolder.setUserRoleCalls).toHaveLength(1); + + const member = await prisma.orgMember.findFirst({ + where: { userId: invitee.id, organizationId: organization.id }, + }); + expect(member).not.toBeNull(); + } + ); +}); + +describe("inviteMembers", () => { + postgresTest( + "skips emails that already belong to a member of the org", + { timeout: 60_000 }, + async ({ prisma }) => { + prismaHolder.client = prisma; + const { inviteMembers } = await import("../app/models/member.server"); + + const { inviter, invitee, organization, invite } = await seedInviteFixture(prisma, { + activeProjectCount: 0, + }); + + await prisma.orgMemberInvite.delete({ where: { id: invite.id } }); + + await prisma.orgMember.create({ + data: { + organizationId: organization.id, + userId: invitee.id, + role: "MEMBER", + }, + }); + + const newcomerEmail = `newcomer-${randomHex(8)}@test.local`; + + const { created, alreadyMembers, alreadyInvited } = await inviteMembers({ + slug: organization.slug, + emails: [invitee.email, newcomerEmail], + userId: inviter.id, + }); + + expect(created.map((row) => row.email)).toEqual([newcomerEmail]); + expect(alreadyMembers).toEqual([invitee.email]); + expect(alreadyInvited).toEqual([]); + + const invites = await prisma.orgMemberInvite.findMany({ + where: { organizationId: organization.id }, + select: { email: true }, + }); + expect(invites.map((pending) => pending.email)).toEqual([newcomerEmail]); + } + ); + + postgresTest( + "reports an email with a pending invitation as already invited", + { timeout: 60_000 }, + async ({ prisma }) => { + prismaHolder.client = prisma; + const { inviteMembers } = await import("../app/models/member.server"); + + const { inviter, organization, invite } = await seedInviteFixture(prisma, { + activeProjectCount: 0, + }); + + const newcomerEmail = `newcomer-${randomHex(8)}@test.local`; + + const { created, alreadyMembers, alreadyInvited } = await inviteMembers({ + slug: organization.slug, + emails: [invite.email, newcomerEmail], + userId: inviter.id, + }); + + expect(created.map((row) => row.email)).toEqual([newcomerEmail]); + expect(alreadyInvited).toEqual([invite.email]); + expect(alreadyMembers).toEqual([]); + } + ); }); describe("provisionMemberDevelopmentEnvironments", () => { From 4eb9292cbeee57304c9ddb3487b1948c6d0e3248 Mon Sep 17 00:00:00 2001 From: Eric Allam <eric@trigger.dev> Date: Wed, 29 Jul 2026 16:45:24 +0100 Subject: [PATCH 31/67] feat(webapp,run-engine): queue metrics and health dashboard (#4131) ## Summary Three related changes, each independently gated: **Queue metrics and health.** Per-queue depth, throughput (enqueued, started, completed), concurrency, whether a queue is throttled, and scheduling delay (how long a run waits between becoming eligible and actually starting), plus a per concurrency-key breakdown for keyed queues. Collected from inside the run queue itself, stored in ClickHouse, and surfaced on the Queues list, a new per-queue detail page, the task pages, and the run inspector. The question it answers is "does this queue have enough concurrency to keep up, and if not, which key or which limit is the constraint". **Percent-based queue concurrency limits.** A queue's concurrency override can now be expressed as a percentage of the environment limit, stored as the source of truth and re-materialized whenever the environment limit changes. Absolute overrides above the environment limit are now **rejected with a 400** instead of being silently capped, which is a behavior change on `POST /api/v1/queues/:queue/concurrency/override`. **The `health` report.** A server-computed verdict on whether work is flowing, whether the runs that do start are healthy, and whether telemetry is fresh, rendered as text with sparklines. Available as `GET /api/v1/reports/:key`, `trigger report`, and the `get_report` MCP tool (plus a `report` MCP prompt, which shows up as a slash command in hosts that support prompts). With the flags off, the Queues page renders the pre-metrics component verbatim, nothing is emitted, and nothing is written to ClickHouse. ## Configuration Two independent gates, on purpose. Emission is global so data accrues for everyone before anyone can look at it; the view is per organization so it can be turned on for one org at a time without a deploy. **Runtime flags (no restart)** | Flag | Store | Gates | | --- | --- | --- | | `queue_metrics:enabled` | run-queue Redis key (`"1"`/`"0"`, off by default) | All emission, gauges and counters. Cached in-process for 10s with stale-while-revalidate, warmed eagerly at boot so the first op after a deploy is not dropped. | | `queue_metrics:gauge_sample_rate` | run-queue Redis key, `0..1` | Fraction of queue ops that emit a gauge. Counters are never sampled, so throughput stays exact at any rate. | | `queueMetricsUiEnabled` | feature-flag catalog: global `FeatureFlag` row, per-org `Organization.featureFlags` override wins | Whether an org sees the metrics view at all: the Queues list variant, the queue detail route, the built-in Queues dashboard, the concurrency-keys endpoint, and the metrics blocks on task pages and the run inspector. Off by default; a gated org gets a 404 on the detail route rather than an empty page. | Both Redis keys are readable and writable from `/admin/queue-metrics` (super-admin UI, with a live per-shard stream-health table) and `GET`/`POST /admin/api/v1/queue-metrics` (admin PAT). The admin surface uses its own Redis client, so it works on any instance regardless of whether that instance runs the emitter or the consumer. **Environment variables (boot time)** | Variable | Default | Notes | | --- | --- | --- | | `QUEUE_METRICS_EMIT_ENABLED` | `0` | Constructs the emitter and injects it into the run engine. Without it the run queue has no emitter at all. | | `QUEUE_METRICS_CONSUMER_ENABLED` | `0` | Boots the stream consumer on this instance. Independent of emission, so consumers can be sized separately from the API. | | `QUEUE_METRICS_STREAM_SHARD_COUNT` | `4` | Stream shards, hashed per queue. | | `QUEUE_METRICS_CONSUMER_BATCH_SIZE` | `1000` | Poll batch equals insert batch, so an ack can never outrun a write. | | `QUEUE_METRICS_REDIS_{HOST,PORT,USERNAME,PASSWORD,TLS_DISABLED}` | falls back to the run-queue Redis | Set `HOST` to move the metrics stream onto a dedicated instance so a metrics backlog cannot compete with the run queue for memory. Self-hosters can leave it unset and get a single-Redis deployment. | | `QUEUE_METRICS_COUNTER_STREAM_MAXLEN` | `2000000` shared, `8000000` dedicated | Bound on how much a stalled consumer can hold. The default is deliberately lower when the stream shares the queue-critical Redis. | | `QUEUE_METRICS_COUNTER_ODOMETER_TTL_SECONDS` | `604800` | TTL on the per-queue cumulative counter key, refreshed on every write, so only queues idle for the whole window are purged. | | `QUEUE_METRICS_MAX_QUEUE_NAMES_PER_ENV` | `1000` | Distinct queue names tracked per environment; overflow collapses into `__overflow__`. | | `QUEUE_METRICS_MAX_CONCURRENCY_KEYS_PER_QUEUE` | `10000` | Same idea one level down, per queue. | | `QUEUE_METRICS_GAUGE_SAMPLE_RATE` | `1` | Default for the live sample-rate key above. | | `QUEUE_METRICS_QUERY_TABLES_VISIBLE` | `0` | Lists the queue-metrics tables in the Query page, its schema docs, the schema API and the AI query context. Off keeps them unlisted while the feature is dark; a query naming them still runs either way. | | `QUEUE_METRICS_CLICKHOUSE_URL` | falls back to the shared wiring | Runs queue metrics on their own ClickHouse service: the consumer's inserts and every queue-metrics read go through it, so a metrics-heavy chart refresh never competes with runs-list or trace reads. Unset reproduces the previous split exactly (inserts on `CLICKHOUSE_URL`, reads on the query pool). | | `QUEUE_METRICS_CLICKHOUSE_READER_URL` | the write URL | Reader split, so the consumer's inserts can never land on a read endpoint. | | `QUEUE_METRICS_CLICKHOUSE_{KEEP_ALIVE_ENABLED,KEEP_ALIVE_IDLE_SOCKET_TTL_MS,MAX_OPEN_CONNECTIONS,LOG_LEVEL,COMPRESSION_REQUEST}` | `1`, unset, `10`, `info`, `1` | Pool tuning, matching the other per-workload ClickHouse clients. | Migrations to apply: ClickHouse `036_create_queue_metrics_v1.sql`, and a Postgres migration adding the nullable `TaskQueue.concurrencyLimitOverridePercent`. Both are additive. ## How collection works Queue operations produce two kinds of signal, and they have opposite failure modes, so they are handled differently. **Gauges** (queued, running, queue limit, env queued, env running, env limit, throttled, plus keys-with-backlog and worst-key wait on keyed queues) are read *inside* the same Redis script that performs the enqueue or dequeue, so the reading is atomic with the operation it describes rather than a racy follow-up read. The script returns them on its reply and the app forwards them to the stream. Gauges are sampled and drop-tolerant: they are aggregated with `max`, so a lost reading costs resolution, never correctness. **Counters** (enqueued, started, completed, plus nack and dead-lettered) are cumulative odometers. Each event increments a per-queue key on the metrics Redis and emits the absolute total, and ClickHouse takes the difference across buckets at read time. This is the important property of the design: a summed-delta counter undercounts permanently on any lost event, while a cumulative one self-heals, because the next surviving reading restates the whole total. Only bucket granularity can be lost, never the total. A queue returning after its odometer TTL expired restarts at 1 and reset detection handles it, which is safe precisely because expiry only spans a window with no activity. Both land on one sharded Redis stream. A consumer reads it with a consumer group, reclaims stale pending entries on a 15s interval rather than on every poll, maps one entry to one or two ClickHouse rows (whole-queue and, for keyed queues, per-key), and acks only after the insert lands. Each batch carries a dedup token derived from its stream-entry ids, and the target tables set `non_replicated_deduplication_window`, so a retried batch cannot double-count either the raw rows or the aggregates that hang off them. Consumer and emitter both emit OTel metrics (`queue_metrics.emitter.emitted`, `queue_metrics.consumer.{entries,rows_inserted,insert_errors,insert_duration,stream_depth,group_lag,pending,lag_unknown}`); stream depth and group lag are the two worth alerting on, and `lag_unknown` exists because Redis can report a null lag after a trim, which must not be read as zero. ## Storage and read path `queue_metrics_raw_v1` is a short landing table with a 6 hour TTL. Four aggregate tiers are materialized straight from raw, never cascaded off each other, each with a 30 day TTL: - `queue_metrics_v1`, 10 second buckets per queue, the default read path - `queue_metrics_5m_v1`, 5 minute buckets per queue, for wide ranges and cross-queue ranking - `env_metrics_v1`, 10 second buckets per environment, queue-independent so it stays cheap at any range - `queue_metrics_ck_v1`, 10 second buckets per concurrency key Every tier is an MV from raw because the counter states do not survive a cascade: their merge is order sensitive, so a `-MergeState` chain off the 10s table inflates the result, and the same property means an aggregate state may only be merged inside one queue. That constraint is now enforced by the query engine rather than by reviewer discipline: a column can declare a `mergeGroupKey`, and any query that references it without grouping by, or pinning to a single value of, every named key fails to compile with an actionable message. On the read side, TRQL gains three tables (`queue_metrics`, `env_metrics`, and a `queue_metrics_by_key` that is hidden from the editor, schema docs and schema API but still queryable, so per-key rows can never silently merge into a plain per-queue query), plus `deltaSumTimestampMerge` and `quantilesTDigestMerge`. Two schema-level optimizations ride along: a table can declare coarser rollups, so a query whose bucket interval is 5 minutes or wider is routed to the 5m table with no change to the query itself, and it can opt into the ClickHouse query cache with time bounds floored to a fixed grid, so the auto-refreshing dashboards actually share cache entries instead of missing on every tick. Both are caller-side substitutions, so the printer stays unaware of physical layout. All of this can also live on its own ClickHouse service. A table declares the pool its reads run on, the three queue-metrics tables name the dedicated one, and the ingestion consumer writes through the same client, so both directions move together with one env var and nothing else routes differently. The other engine change is opt-in gap filling: charts can request rows for empty buckets, where counters zero-fill and gauges carry forward. Grouped gauge series are densified per group and carried inside a partition, so a quiet queue's line holds its last value without bleeding another queue's value into it. ## Queue concurrency limits `concurrencyLimitOverridePercent` on `TaskQueue` is the source of truth when an override is set as a percentage; the absolute `concurrencyLimit` is materialized from it (floored, clamped to at least 1 so a percentage can never act as a pause, and never above the environment limit). Every path that changes an environment limit now recalculates the environment's percent-based overrides afterwards, outside the transaction, and pushes changed limits to the engine. The push is attempted even when the stored value did not change, so a previously failed sync self-heals rather than leaving the database and the engine diverged; paused queues are skipped so a recalculation cannot effectively unpause one. The API accepts exactly one of `concurrencyLimit` or `percent`, and the reject-instead-of-clamp change above means a request asking for more than the environment allows now fails loudly. The percent bound (greater than 0, at most 100) is defined once and shared by the zod schema, the dashboard mutation handler and the service, so the three cannot drift. The concurrency-keys table on a queue is now paginated against the ClickHouse per-key tier, ranked by peak backlog with the total on every row from a single scan, and only the keys on the current page are enriched with live counts from Redis. That replaces a hard top-50 cap with something whose cost is a function of page size rather than key cardinality. ## The health report `GET /api/v1/reports/:key?period=&format=markdown|ansi|json`. The verdict is computed on the server and is deterministic, not model-generated. Three independent analyzers run over one input snapshot: flow (is work moving, and if not, is the cause a limit, throttling, one bad queue, or dead-lettering), execution (are the runs that start succeeding, and at what latency), and liveness (how fresh is the telemetry). When telemetry is genuinely stale, the first two are forced to unknown and every actionable field is stripped, so no surface ever advises action off stale data. Authorization is per query table rather than a blanket query grant: a JWT must be scoped to every table the report reads (`runs`, `env_metrics`, `queue_metrics`), so a narrowly scoped token cannot pull a report that reads more than it was granted. `period` is validated as a shorthand with a 90 day ceiling at the edge. The report catalog is a registry of `{ load, interpret }` entries, so the next report is a new entry and no change to the route, the view model, the renderers, the CLI or the MCP tool. `trigger mcp` no longer launches the install wizard when stdout is a TTY, which fixed a real failure: hosts spawn the server over a PTY, so the wizard would open and the client would time out waiting for a server that never started. The wizard now needs `trigger mcp --install`. ## The part that is live regardless of every flag The enqueue and dequeue scripts now return a 2-tuple so a gauge reading can ride back on the reply. Every return site in the eight affected scripts is wrapped, and a `nil` original is converted to `false` on the way out, because a raw `nil` in the first slot would make Lua truncate the multi-bulk reply and silently drop the gauge on the throttled and empty-queue paths. The reply shape and the destructuring on the app side are exercised on every queue operation whether or not metrics are enabled, so that is the part of `run-engine` worth the closest review. One behavior fix in the same area: the scheduling-delay anchor is set only on a run's first entry into the queue. Anchoring it to trigger time on re-enqueues made waitpoint and checkpoint resumes report the entire wait as scheduling delay. Queue ordering is untouched, so a re-enqueued run keeps its position, and nacks deliberately keep the original anchor because a rolled-back dequeue is the same continuous wait. A pending-version promotion still anchors to trigger time, on purpose: that promotion is the run's first real entry into the queue, since the trigger deliberately held it back waiting for a worker version, and the TTL is armed at the same point for the same reason. The consequence is worth naming, because it is a judgement call: a run that waits on a deployment reports that wait as scheduling delay on its queue, which is time unrelated to queue capacity. ## Verification Unit and integration suites across the new package, the run queue, the mapping layer, the query engine and ClickHouse (including a test that applies migration 036 through the same splitter CI uses, and a regression test that inserts the same batch three times to prove the aggregates do not inflate). Beyond that, the whole path was driven end to end against a live stack with real runs: emitter to Redis stream to consumer to ClickHouse to the dashboards, for both the local dev path and the deployed path where a supervisor drives the dequeue, with assertions on exact counter reconstruction per queue and per concurrency key, throttling, environment saturation, scheduling delay, and a deliberate mid-stream reading drop to confirm the cumulative counters still reconstruct the correct total. The gated-off state was checked on every touched surface. The dedicated ClickHouse service was verified against a second, separately-schema'd instance: with it configured, the driven counters reconstruct exactly on the dedicated instance, the shared instance gains no rows for that window, a read through the query API returns the value that exists only on the dedicated instance, and a `runs` query still succeeds (it would fail outright if it were mis-routed to a service without that table). With the variable unset, the full suite passes unchanged. --------- Co-authored-by: Katia Bulatova <katia@trigger.dev> Co-authored-by: Katia Bulatova <katherine.bulatova@gmail.com> Co-authored-by: James Ritchie <james@trigger.dev> --- .../mcp-server-starts-without-wizard.md | 5 + .changeset/report-health.md | 12 + .../agent-detail-metrics-layout.md | 6 + .../paginate-concurrency-keys-table.md | 6 + .../queue-concurrency-percent-limits.md | 6 + .server-changes/queue-metrics-dashboard.md | 6 + .../reject-queue-limit-above-env-limit.md | 6 + .../app/components/layout/MetricsLayout.tsx | 368 ++++ .../components/metrics/ActivityBarChart.tsx | 82 + .../app/components/metrics/BigNumber.tsx | 12 +- .../app/components/metrics/MiniLineChart.tsx | 222 ++ .../app/components/primitives/Buttons.tsx | 10 + .../primitives/SegmentedControl.tsx | 6 +- .../app/components/primitives/Table.tsx | 176 +- .../app/components/primitives/Tooltip.tsx | 2 +- .../components/primitives/TooltipPortal.tsx | 24 +- .../components/primitives/UsageSparkline.tsx | 24 +- .../app/components/primitives/charts/Card.tsx | 2 +- .../components/primitives/charts/Chart.tsx | 7 +- .../components/primitives/charts/ChartBar.tsx | 2 +- .../primitives/charts/ChartCard.tsx | 17 +- .../primitives/charts/ChartLine.tsx | 515 ++++- .../primitives/charts/ChartSyncContext.tsx | 3 + .../app/components/primitives/useTableSort.ts | 130 ++ .../app/components/query/QueryEditor.tsx | 11 +- .../app/components/queues/QueueControls.tsx | 396 ++++ .../components/queues/QueueMetricCards.tsx | 462 ++++ apps/webapp/app/entry.server.tsx | 3 + apps/webapp/app/env.server.ts | 47 + apps/webapp/app/features.server.ts | 2 + apps/webapp/app/hooks/useFeatures.ts | 8 +- .../app/hooks/useMetricResourceQuery.ts | 165 ++ .../webapp/app/models/queueMutation.server.ts | 158 ++ .../presenters/v3/BuiltInDashboards.server.ts | 181 +- .../v3/MetricDashboardPresenter.server.ts | 3 + .../v3/QueueAllocationPresenter.server.ts | 56 + .../v3/QueueListPresenter.server.ts | 203 +- .../v3/QueueMetricsPresenter.server.ts | 145 ++ .../v3/QueueRetrievePresenter.server.ts | 34 +- .../v3/RunQueueMetricsPresenter.server.ts | 174 ++ .../v3/reports/ReportPresenter.server.ts | 49 + .../presenters/v3/reports/health/execution.ts | 55 + .../app/presenters/v3/reports/health/flow.ts | 304 +++ .../v3/reports/health/health-core.ts | 271 +++ .../v3/reports/health/health-data.ts | 644 ++++++ .../v3/reports/health/health-messages.ts | 170 ++ .../presenters/v3/reports/health/health.ts | 268 +++ .../presenters/v3/reports/health/liveness.ts | 27 + .../presenters/v3/reports/renderMarkdown.ts | 522 +++++ .../presenters/v3/reports/report-messages.ts | 39 + .../presenters/v3/reports/report-registry.ts | 37 + .../v3/reports/report-view-model.ts | 226 ++ .../route.tsx | 79 +- .../route.tsx | 340 ++- .../route.tsx | 11 + .../ExamplesContent.tsx | 18 +- .../TableSchemaContent.tsx | 20 +- .../route.tsx | 1870 +++++++++++++---- .../route.tsx | 1197 +++++++++++ .../route.tsx | 73 +- .../route.tsx | 153 +- .../route.tsx | 333 ++- ...dmin.api.v1.environments.$environmentId.ts | 4 + ...api.v1.orgs.$organizationId.concurrency.ts | 7 + .../app/routes/admin.api.v1.queue-metrics.ts | 45 + .../webapp/app/routes/admin.queue-metrics.tsx | 190 ++ apps/webapp/app/routes/admin.tsx | 4 + apps/webapp/app/routes/api.v1.query.schema.ts | 7 +- ...queues.$queueParam.concurrency.override.ts | 29 +- apps/webapp/app/routes/api.v1.reports.$key.ts | 93 + apps/webapp/app/routes/resources.metric.tsx | 4 + ...tParam.env.$envParam.query.ai-generate.tsx | 6 +- .../route.tsx | 264 ++- .../resources.queues.concurrency-keys.ts | 180 ++ .../app/routes/storybook.charts/route.tsx | 152 ++ .../app/routes/storybook.layout/route.tsx | 373 ++++ apps/webapp/app/routes/storybook/route.tsx | 4 + .../clickhouse/clickhouseFactory.server.ts | 78 +- .../app/services/queryService.server.ts | 108 +- apps/webapp/app/utils/pathBuilder.ts | 9 + .../app/v3/canAccessQueueMetricsUi.server.ts | 26 + apps/webapp/app/v3/featureFlags.ts | 4 + apps/webapp/app/v3/querySchemas.ts | 492 ++++- apps/webapp/app/v3/queueMetrics.server.ts | 247 +++ apps/webapp/app/v3/queueMetricsMapping.ts | 164 ++ apps/webapp/app/v3/queueSparklineGrid.ts | 54 + apps/webapp/app/v3/runEngine.server.ts | 2 + .../v3/services/allocateConcurrency.server.ts | 9 + .../v3/services/concurrencySystem.server.ts | 148 +- apps/webapp/package.json | 1 + .../concurrencySystemPercentOverride.test.ts | 370 ++++ apps/webapp/test/queueMetricsMapping.test.ts | 239 +++ apps/webapp/test/queueSparklineGrid.test.ts | 70 + apps/webapp/test/reportHealth.test.ts | 511 +++++ apps/webapp/test/reportHealthData.test.ts | 242 +++ apps/webapp/test/useTableSort.test.ts | 55 + docs/v3-openapi.yaml | 9 +- .../schema/036_create_queue_metrics_v1.sql | 267 +++ .../clickhouse/src/client/tsql.ts | 6 + internal-packages/clickhouse/src/index.ts | 22 + .../clickhouse/src/queueMetrics.test.ts | 525 +++++ .../clickhouse/src/queueMetrics.ts | 294 +++ .../migration.sql | 5 + .../database/prisma/schema.prisma | 14 +- .../metrics-pipeline/package.json | 33 + .../metrics-pipeline/src/cachedValue.ts | 125 ++ .../metrics-pipeline/src/consumer.test.ts | 392 ++++ .../metrics-pipeline/src/consumer.ts | 336 +++ .../metrics-pipeline/src/emitter.ts | 242 +++ .../metrics-pipeline/src/flag.ts | 46 + .../metrics-pipeline/src/hash.ts | 15 + .../metrics-pipeline/src/idempotency.ts | 11 + .../metrics-pipeline/src/index.ts | 26 + internal-packages/metrics-pipeline/src/lua.ts | 50 + .../metrics-pipeline/src/pipeline.test.ts | 116 + .../metrics-pipeline/src/types.ts | 42 + .../metrics-pipeline/test/setup.ts | 4 + .../metrics-pipeline/tsconfig.build.json | 21 + .../metrics-pipeline/tsconfig.json | 8 + .../metrics-pipeline/tsconfig.src.json | 20 + .../metrics-pipeline/tsconfig.test.json | 21 + .../metrics-pipeline/vitest.config.ts | 17 + internal-packages/run-engine/package.json | 1 + .../run-engine/src/engine/index.ts | 39 +- .../src/engine/systems/delayedRunSystem.ts | 1 + .../src/engine/systems/enqueueSystem.ts | 27 +- .../src/engine/tests/pendingVersion.test.ts | 18 +- .../run-engine/src/engine/tests/ttl.test.ts | 9 + .../run-engine/src/engine/types.ts | 3 + .../run-engine/src/run-queue/index.ts | 427 +++- .../run-engine/src/run-queue/keyProducer.ts | 14 +- .../run-engine/src/run-queue/metrics.test.ts | 397 ++++ .../src/run-queue/tests/ckIndex.test.ts | 42 + .../run-engine/src/run-queue/types.ts | 3 + internal-packages/tsql/src/index.test.ts | 20 + internal-packages/tsql/src/index.ts | 25 + internal-packages/tsql/src/query/functions.ts | 13 + .../tsql/src/query/printer.test.ts | 385 ++++ internal-packages/tsql/src/query/printer.ts | 342 ++- .../tsql/src/query/printer_context.ts | 16 +- internal-packages/tsql/src/query/schema.ts | 32 + packages/cli-v3/src/apiClient.ts | 43 + packages/cli-v3/src/cli/index.ts | 2 + packages/cli-v3/src/commands/mcp.ts | 13 +- packages/cli-v3/src/commands/report.ts | 111 + packages/cli-v3/src/mcp/config.ts | 6 + packages/cli-v3/src/mcp/prompts.ts | 64 + packages/cli-v3/src/mcp/schemas.ts | 44 + packages/cli-v3/src/mcp/tools.ts | 6 +- packages/cli-v3/src/mcp/tools/report.ts | 45 + packages/core/src/v3/apiClient/index.ts | 49 + pnpm-lock.yaml | 25 + 152 files changed, 18359 insertions(+), 1124 deletions(-) create mode 100644 .changeset/mcp-server-starts-without-wizard.md create mode 100644 .changeset/report-health.md create mode 100644 .server-changes/agent-detail-metrics-layout.md create mode 100644 .server-changes/paginate-concurrency-keys-table.md create mode 100644 .server-changes/queue-concurrency-percent-limits.md create mode 100644 .server-changes/queue-metrics-dashboard.md create mode 100644 .server-changes/reject-queue-limit-above-env-limit.md create mode 100644 apps/webapp/app/components/layout/MetricsLayout.tsx create mode 100644 apps/webapp/app/components/metrics/ActivityBarChart.tsx create mode 100644 apps/webapp/app/components/metrics/MiniLineChart.tsx create mode 100644 apps/webapp/app/components/primitives/useTableSort.ts create mode 100644 apps/webapp/app/components/queues/QueueControls.tsx create mode 100644 apps/webapp/app/components/queues/QueueMetricCards.tsx create mode 100644 apps/webapp/app/hooks/useMetricResourceQuery.ts create mode 100644 apps/webapp/app/models/queueMutation.server.ts create mode 100644 apps/webapp/app/presenters/v3/QueueAllocationPresenter.server.ts create mode 100644 apps/webapp/app/presenters/v3/QueueMetricsPresenter.server.ts create mode 100644 apps/webapp/app/presenters/v3/RunQueueMetricsPresenter.server.ts create mode 100644 apps/webapp/app/presenters/v3/reports/ReportPresenter.server.ts create mode 100644 apps/webapp/app/presenters/v3/reports/health/execution.ts create mode 100644 apps/webapp/app/presenters/v3/reports/health/flow.ts create mode 100644 apps/webapp/app/presenters/v3/reports/health/health-core.ts create mode 100644 apps/webapp/app/presenters/v3/reports/health/health-data.ts create mode 100644 apps/webapp/app/presenters/v3/reports/health/health-messages.ts create mode 100644 apps/webapp/app/presenters/v3/reports/health/health.ts create mode 100644 apps/webapp/app/presenters/v3/reports/health/liveness.ts create mode 100644 apps/webapp/app/presenters/v3/reports/renderMarkdown.ts create mode 100644 apps/webapp/app/presenters/v3/reports/report-messages.ts create mode 100644 apps/webapp/app/presenters/v3/reports/report-registry.ts create mode 100644 apps/webapp/app/presenters/v3/reports/report-view-model.ts create mode 100644 apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx create mode 100644 apps/webapp/app/routes/admin.api.v1.queue-metrics.ts create mode 100644 apps/webapp/app/routes/admin.queue-metrics.tsx create mode 100644 apps/webapp/app/routes/api.v1.reports.$key.ts create mode 100644 apps/webapp/app/routes/resources.queues.concurrency-keys.ts create mode 100644 apps/webapp/app/routes/storybook.layout/route.tsx create mode 100644 apps/webapp/app/v3/canAccessQueueMetricsUi.server.ts create mode 100644 apps/webapp/app/v3/queueMetrics.server.ts create mode 100644 apps/webapp/app/v3/queueMetricsMapping.ts create mode 100644 apps/webapp/app/v3/queueSparklineGrid.ts create mode 100644 apps/webapp/test/concurrencySystemPercentOverride.test.ts create mode 100644 apps/webapp/test/queueMetricsMapping.test.ts create mode 100644 apps/webapp/test/queueSparklineGrid.test.ts create mode 100644 apps/webapp/test/reportHealth.test.ts create mode 100644 apps/webapp/test/reportHealthData.test.ts create mode 100644 apps/webapp/test/useTableSort.test.ts create mode 100644 internal-packages/clickhouse/schema/036_create_queue_metrics_v1.sql create mode 100644 internal-packages/clickhouse/src/queueMetrics.test.ts create mode 100644 internal-packages/clickhouse/src/queueMetrics.ts create mode 100644 internal-packages/database/prisma/migrations/20260716120000_add_task_queue_concurrency_limit_override_percent/migration.sql create mode 100644 internal-packages/metrics-pipeline/package.json create mode 100644 internal-packages/metrics-pipeline/src/cachedValue.ts create mode 100644 internal-packages/metrics-pipeline/src/consumer.test.ts create mode 100644 internal-packages/metrics-pipeline/src/consumer.ts create mode 100644 internal-packages/metrics-pipeline/src/emitter.ts create mode 100644 internal-packages/metrics-pipeline/src/flag.ts create mode 100644 internal-packages/metrics-pipeline/src/hash.ts create mode 100644 internal-packages/metrics-pipeline/src/idempotency.ts create mode 100644 internal-packages/metrics-pipeline/src/index.ts create mode 100644 internal-packages/metrics-pipeline/src/lua.ts create mode 100644 internal-packages/metrics-pipeline/src/pipeline.test.ts create mode 100644 internal-packages/metrics-pipeline/src/types.ts create mode 100644 internal-packages/metrics-pipeline/test/setup.ts create mode 100644 internal-packages/metrics-pipeline/tsconfig.build.json create mode 100644 internal-packages/metrics-pipeline/tsconfig.json create mode 100644 internal-packages/metrics-pipeline/tsconfig.src.json create mode 100644 internal-packages/metrics-pipeline/tsconfig.test.json create mode 100644 internal-packages/metrics-pipeline/vitest.config.ts create mode 100644 internal-packages/run-engine/src/run-queue/metrics.test.ts create mode 100644 packages/cli-v3/src/commands/report.ts create mode 100644 packages/cli-v3/src/mcp/prompts.ts create mode 100644 packages/cli-v3/src/mcp/tools/report.ts diff --git a/.changeset/mcp-server-starts-without-wizard.md b/.changeset/mcp-server-starts-without-wizard.md new file mode 100644 index 00000000000..72a1a8bc659 --- /dev/null +++ b/.changeset/mcp-server-starts-without-wizard.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +`trigger mcp` now always starts the MCP server, and the interactive install wizard has moved behind `trigger mcp --install`. Previously the wizard opened whenever stdout was a terminal, so any MCP host that spawns the command over a pseudo-terminal waited on a server that never started and eventually timed out. diff --git a/.changeset/report-health.md b/.changeset/report-health.md new file mode 100644 index 00000000000..51f06733c7f --- /dev/null +++ b/.changeset/report-health.md @@ -0,0 +1,12 @@ +--- +"@trigger.dev/core": patch +"trigger.dev": patch +--- + +Ask whether an environment is healthy and get an answer instead of a wall of charts. `trigger report health` returns a verdict on three questions: is work flowing, are the runs that start succeeding, and is the telemetry fresh enough to trust either answer. When something looks wrong it names the most likely cause and a next action. + +```bash +npx trigger.dev@latest report health --env prod --period 24h +``` + +The verdict is computed server side, so the CLI, the new `get_report` MCP tool, and `GET /api/v1/reports/health` all return the same text with the same sparklines. In MCP hosts that support prompts, `report` is also available as a slash command. diff --git a/.server-changes/agent-detail-metrics-layout.md b/.server-changes/agent-detail-metrics-layout.md new file mode 100644 index 00000000000..1db02286f09 --- /dev/null +++ b/.server-changes/agent-detail-metrics-layout.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +The agent detail page now uses the same layout as the other metrics pages: filters stay pinned at the top, the activity charts sit in a row beneath them, and the tabs and table scroll with the page, with the agent config in a resizable panel on the right. diff --git a/.server-changes/paginate-concurrency-keys-table.md b/.server-changes/paginate-concurrency-keys-table.md new file mode 100644 index 00000000000..3f9c610362d --- /dev/null +++ b/.server-changes/paginate-concurrency-keys-table.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +A queue's concurrency keys are now paged and searchable, so a queue with thousands of keys shows all of them instead of only the busiest 50. diff --git a/.server-changes/queue-concurrency-percent-limits.md b/.server-changes/queue-concurrency-percent-limits.md new file mode 100644 index 00000000000..5e4d70c5433 --- /dev/null +++ b/.server-changes/queue-concurrency-percent-limits.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: feature +--- + +A queue's concurrency limit can now be set as a percentage of your environment limit, from the dashboard or the API. Percentage limits track the environment limit, so raising or lowering it re-divides capacity without you editing every queue. diff --git a/.server-changes/queue-metrics-dashboard.md b/.server-changes/queue-metrics-dashboard.md new file mode 100644 index 00000000000..4892040a639 --- /dev/null +++ b/.server-changes/queue-metrics-dashboard.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: feature +--- + +See how every queue in an environment is doing: backlog, throughput, concurrency against its limit, when it was throttled, and how long runs waited before starting. Each queue also gets its own page, which breaks the same numbers down per concurrency key so you can tell whether one key is starving the rest. diff --git a/.server-changes/reject-queue-limit-above-env-limit.md b/.server-changes/reject-queue-limit-above-env-limit.md new file mode 100644 index 00000000000..dcd1dad89f8 --- /dev/null +++ b/.server-changes/reject-queue-limit-above-env-limit.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: breaking +--- + +Setting a queue concurrency limit higher than your environment limit is now rejected with an error instead of being silently reduced to the environment limit, so a queue never looks like it has more capacity than it can get. Limits already saved are unchanged. diff --git a/apps/webapp/app/components/layout/MetricsLayout.tsx b/apps/webapp/app/components/layout/MetricsLayout.tsx new file mode 100644 index 00000000000..12d0fcd743c --- /dev/null +++ b/apps/webapp/app/components/layout/MetricsLayout.tsx @@ -0,0 +1,368 @@ +/** + * MetricsLayout — a compound layout for metric / dashboard pages. + * + * Slots bake all the chrome; there is no `className` on any slot, so pages can't drift apart on + * spacing. Need a variant? Add a closed prop (`kind`, `inset`), don't reopen `className`. + * + * Slots, top to bottom: + * - `Filters` — pinned 40px bar under the NavBar. Left/right clusters are child divs. + * - `Grid` — tiles; columns derived from tile count unless `columns` is set. `kind="charts"` + * bakes the fixed chart-row height. + * - `Content` — table / tabs below the tiles. Full-bleed by default; `inset` for a padded column. + * + * Optional: + * - `Sidebar` — a persistent right-hand panel; fixed `width` or `resizable`. Present ⇒ Root + * switches to `[main | sidebar]`; absent ⇒ single column. + * - `scroll` on Root — `"page"` (default): the whole page scrolls as one. `"regions"`: Root owns + * no scroll, the page composes its own scrolling areas. + * + * Purely presentational. Lives inside a `PageContainer` after the `NavBar`. + * + * @example + * ```tsx + * <MetricsLayout.Root> + * <MetricsLayout.Filters> + * <div className="flex items-center gap-2">…search + TimeFilter…</div> + * <PaginationControls … /> + * </MetricsLayout.Filters> + * <MetricsLayout.Grid>…stat tiles…</MetricsLayout.Grid> + * <MetricsLayout.Grid kind="charts">…chart tiles…</MetricsLayout.Grid> + * <MetricsLayout.Content>…table…</MetricsLayout.Content> + * </MetricsLayout.Root> + * ``` + */ +import { Children, isValidElement, type ReactElement, type ReactNode } from "react"; +import { PageBody } from "~/components/layout/AppLayout"; +import { + ResizableHandle, + ResizablePanel, + ResizablePanelGroup, + type ResizableSnapshot, +} from "~/components/primitives/Resizable"; +import { cn } from "~/utils/cn"; + +type ColumnCount = 1 | 2 | 3 | 4 | 5 | 6; + +/** + * A responsive column spec. Each key is a breakpoint (mobile-first `base`, then `sm`/`md`/`lg`); + * the value is the number of grid columns from that breakpoint up. Pass this to `Grid` when the + * tile count shouldn't drive the layout (e.g. a chart grid that is always two-up). + */ +export type GridColumns = { + base?: ColumnCount; + sm?: ColumnCount; + md?: ColumnCount; + lg?: ColumnCount; +}; + +// Static class maps so Tailwind's scanner sees every column class as a literal string. +const BASE_COLS: Record<ColumnCount, string> = { + 1: "grid-cols-1", + 2: "grid-cols-2", + 3: "grid-cols-3", + 4: "grid-cols-4", + 5: "grid-cols-5", + 6: "grid-cols-6", +}; +const SM_COLS: Record<ColumnCount, string> = { + 1: "sm:grid-cols-1", + 2: "sm:grid-cols-2", + 3: "sm:grid-cols-3", + 4: "sm:grid-cols-4", + 5: "sm:grid-cols-5", + 6: "sm:grid-cols-6", +}; +const MD_COLS: Record<ColumnCount, string> = { + 1: "md:grid-cols-1", + 2: "md:grid-cols-2", + 3: "md:grid-cols-3", + 4: "md:grid-cols-4", + 5: "md:grid-cols-5", + 6: "md:grid-cols-6", +}; +const LG_COLS: Record<ColumnCount, string> = { + 1: "lg:grid-cols-1", + 2: "lg:grid-cols-2", + 3: "lg:grid-cols-3", + 4: "lg:grid-cols-4", + 5: "lg:grid-cols-5", + 6: "lg:grid-cols-6", +}; + +// When `columns` is omitted the grid figures itself out from the tile count. The breakpoints are +// chosen so the common metric layouts fall out for free: a trio of stat blocks goes one-up then +// three-up, a quartet of stat/chart tiles goes two-up then four-up, and anything larger settles +// into a comfortable two-up. +function columnsForCount(count: number): GridColumns { + switch (count) { + case 1: + return { base: 1 }; + case 2: + return { base: 1, sm: 2 }; + case 3: + return { base: 1, sm: 3 }; + case 4: + return { base: 2, lg: 4 }; + default: + return { base: 1, sm: 2 }; + } +} + +/** + * Who owns the vertical scroll. + * - `"page"` (default): Root owns one `overflow-y-auto` and the column rhythm — the whole page + * scrolls as one. + * - `"regions"`: Root only bounds the height (a bare `flex` column, no scroll, no rhythm); the + * page composes its own scrolling areas inside the slots. + */ +export type MetricsScroll = "page" | "regions"; + +/** A length the resizable panels accept: pixels or percent (the panel library's `Unit`). */ +type PanelLength = `${number}px` | `${number}%`; + +type MetricsLayoutSidebarProps = { + children: ReactNode; + /** + * Fixed sidebar width for the non-resizable default (any CSS length, e.g. `"380px"`, `"22rem"`). + * Ignored when `resizable` is set. Defaults to `"380px"`. + */ + width?: string; + /** + * Makes the split draggable. To persist it, pass an `autosaveId` (written to a cookie) plus the + * `snapshot` read back in the loader via `getResizableSnapshot(request, autosaveId)`. + */ + resizable?: boolean; + /** Resizable only: min width of the sidebar panel. Defaults to `"280px"`. */ + min?: PanelLength; + /** Resizable only: initial width of the sidebar panel. Defaults to `"380px"`. */ + defaultSize?: PanelLength; + /** Resizable only: max width of the sidebar panel. */ + max?: PanelLength; + /** Resizable only: min width of the main panel. Defaults to `"300px"`. */ + mainMin?: PanelLength; + /** Resizable only: cookie name the split is persisted under (also the panel-group id). */ + autosaveId?: string; + /** Resizable only: server-loaded split snapshot to hydrate from (see `getResizableSnapshot`). */ + snapshot?: ResizableSnapshot; +}; + +/** + * Marker slot for the persistent side panel. Rendered/positioned entirely by `Root` (this + * component is never mounted directly) — Root reads its props to build the `[main | sidebar]` + * layout and drops the children into the panel. The panel itself owns its chrome (border, scroll); + * pass those as part of the children, not as a class on the slot. + */ +function MetricsLayoutSidebar(_props: MetricsLayoutSidebarProps) { + return null; +} + +function isSidebarElement(child: ReactNode): child is ReactElement<MetricsLayoutSidebarProps> { + return isValidElement(child) && child.type === MetricsLayoutSidebar; +} + +function isFiltersElement(child: ReactNode): child is ReactElement { + return isValidElement(child) && child.type === MetricsLayoutFilters; +} + +// The main (left) column. Filters is hoisted out of the scroll container so it stays pinned while +// the rest scrolls. `"page"` scrolls as one with the baked column rhythm; `"regions"` stays bare so +// the page owns its own scrolling. +function MetricsLayoutMain({ children, scroll }: { children: ReactNode; scroll: MetricsScroll }) { + const arr = Children.toArray(children); + const filters = arr.find(isFiltersElement); + const rest = filters ? arr.filter((child) => !isFiltersElement(child)) : children; + + return ( + <div className="flex h-full min-h-0 flex-col"> + {filters} + <div + className={ + scroll === "page" + ? "flex min-h-0 flex-1 flex-col gap-2.5 overflow-y-auto py-2.5 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-surface-control" + : "flex min-h-0 flex-1 flex-col overflow-hidden" + } + > + {rest} + </div> + </div> + ); +} + +function MetricsLayoutRoot({ + children, + scroll = "page", +}: { + children: ReactNode; + /** Who owns the vertical scroll — see {@link MetricsScroll}. Defaults to `"page"`. */ + scroll?: MetricsScroll; +}) { + // A single optional Sidebar slot flips Root into a horizontal `[main | sidebar]` layout. When it + // is absent the output is the plain single-column markup. + const sidebar = Children.toArray(children).find(isSidebarElement); + const mainChildren = sidebar + ? Children.toArray(children).filter((child) => !isSidebarElement(child)) + : children; + + const main = <MetricsLayoutMain scroll={scroll}>{mainChildren}</MetricsLayoutMain>; + + if (!sidebar) { + return ( + <PageBody scrollable={false}> + {/* The whole page scrolls as one: filters (pinned) aside, the tiles and content share a + single vertical scroll context. */} + {main} + </PageBody> + ); + } + + const { + children: sidebarChildren, + width = "380px", + resizable, + min = "280px", + defaultSize = "380px", + max, + mainMin = "300px", + autosaveId, + snapshot, + } = sidebar.props; + + if (resizable) { + // Draggable split. `autosaveId`/`snapshot` wire up cookie persistence exactly as the run and + // agent pages do (client writes the cookie, the loader hydrates via getResizableSnapshot). + return ( + <PageBody scrollable={false}> + <ResizablePanelGroup + orientation="horizontal" + className="h-full max-h-full" + autosaveId={autosaveId} + snapshot={snapshot} + > + <ResizablePanel id="metrics-main" min={mainMin}> + {main} + </ResizablePanel> + <ResizableHandle id={`${autosaveId ?? "metrics"}-sidebar-handle`} /> + <ResizablePanel + id="metrics-sidebar" + min={min} + default={defaultSize} + max={max} + isStaticAtRest + className="h-full overflow-hidden" + > + {sidebarChildren} + </ResizablePanel> + </ResizablePanelGroup> + </PageBody> + ); + } + + // Fixed-width sidebar. + return ( + <PageBody scrollable={false}> + <div className="flex h-full w-full overflow-hidden"> + <div className="min-w-0 flex-1">{main}</div> + <div className="h-full shrink-0 overflow-hidden" style={{ width }}> + {sidebarChildren} + </div> + </div> + </PageBody> + ); +} + +/** + * The pinned bar under the NavBar. Baked chrome: a 40px-tall bar with a bottom border and the + * standard page insets. Compose left/right clusters as child divs — `justify-between` spreads them + * (a single child sits at the start). + */ +function MetricsLayoutFilters({ + children, + className, +}: { + children: ReactNode; + /** Override the baked horizontal padding (the two queue pages want slightly different insets). */ + className?: string; +}) { + return ( + <div + className={cn( + "flex h-10 shrink-0 items-center justify-between gap-2 border-b border-grid-dimmed pl-2.5 pr-3", + className + )} + > + {children} + </div> + ); +} + +/** Whether a grid holds stat tiles (auto height) or charts (a fixed row height). */ +export type MetricsGridKind = "tiles" | "charts"; + +/** + * A grid of tiles with the baked page gutter and grid gap. Columns are derived from the tile count + * unless you pass an explicit `columns` spec. Pass `kind="charts"` for a row of chart cards — it + * bakes the fixed chart-row height so the cards fill it (no wrapper needed). + */ +function MetricsLayoutGrid({ + children, + columns, + kind = "tiles", +}: { + children: ReactNode; + /** Explicit responsive columns. Omit to derive the layout from the number of tiles. */ + columns?: GridColumns; + /** `"tiles"` (default) sizes to content; `"charts"` bakes the fixed chart-row height. */ + kind?: MetricsGridKind; +}) { + const resolved = columns ?? columnsForCount(Children.toArray(children).length); + return ( + <div + className={cn( + "grid gap-2.5 px-2.5", + // `shrink-0` is load-bearing: the grid sits in Root's flex-col scroll container, where the + // default flex-shrink would collapse a fixed-height row whose chart cards have ~no + // intrinsic height. Pin it so the charts keep their row height and the page scrolls past. + kind === "charts" && "h-[280px] shrink-0", + resolved.base && BASE_COLS[resolved.base], + resolved.sm && SM_COLS[resolved.sm], + resolved.md && MD_COLS[resolved.md], + resolved.lg && LG_COLS[resolved.lg] + )} + > + {children} + </div> + ); +} + +/** + * The content region below the tiles (tabs / table / list). Full-bleed by default so a list table + * spans edge to edge with its own top border; pass `inset` for a padded column (the detail page's + * tabs + charts). Separation from the tiles above comes from the scroll column's gap alone (no + * extra top margin), so the tile → content step matches the gap between tile rows. + */ +function MetricsLayoutContent({ + children, + inset = false, +}: { + children: ReactNode; + /** Pad the content into a column (page gutter) instead of letting it span edge to edge. */ + inset?: boolean; +}) { + return <div className={cn("flex flex-col gap-2.5", inset && "px-2.5")}>{children}</div>; +} + +export const MetricsLayout = { + Root: MetricsLayoutRoot, + Filters: MetricsLayoutFilters, + Grid: MetricsLayoutGrid, + Content: MetricsLayoutContent, + Sidebar: MetricsLayoutSidebar, +}; + +export { + MetricsLayoutRoot, + MetricsLayoutFilters, + MetricsLayoutGrid, + MetricsLayoutContent, + MetricsLayoutSidebar, +}; diff --git a/apps/webapp/app/components/metrics/ActivityBarChart.tsx b/apps/webapp/app/components/metrics/ActivityBarChart.tsx new file mode 100644 index 00000000000..4e361b224b8 --- /dev/null +++ b/apps/webapp/app/components/metrics/ActivityBarChart.tsx @@ -0,0 +1,82 @@ +import { type ReactElement, type ReactNode } from "react"; +import { BarChart, ReferenceLine, Tooltip, YAxis } from "recharts"; +import { SimpleTooltip } from "~/components/primitives/Tooltip"; + +// Fixed px dims skip ResponsiveContainer's ResizeObserver — otherwise every panel resize +// re-renders all the charts in a list at once. +export const ACTIVITY_CHART_WIDTH = 112; +export const ACTIVITY_CHART_HEIGHT = 24; +export const ACTIVITY_CHART_PEAK_CLASS = + "-mt-1 inline-block min-w-7 text-xxs tabular-nums text-text-dimmed"; + +type ActivityBarChartProps = { + /** Recharts row data; each row is a bucket. The bar `children` read their `dataKey`s off it. */ + data: ReadonlyArray<Record<string, unknown>>; + /** Y-axis domain top and the height of the dashed peak line. */ + max: number; + /** The `<Bar>` element(s) — one stacked series per bar, or a single bar with `<Cell>`s. */ + children: ReactNode; + /** Recharts `<Tooltip content>` element for the per-bucket hover card. */ + tooltip: ReactElement; + /** Trailing peak label shown to the right of the chart. */ + peak: ReactNode; + /** Optional tooltip wrapping the peak label. */ + peakTooltip?: ReactNode; + /** Chart width in px. Defaults to the shared ACTIVITY_CHART_WIDTH. */ + width?: number; +}; + +/** + * Shared visual frame for the inline activity/backlog mini bar charts (tasks page + queues list). + * Owns the fixed dimensions, y-axis, hover tooltip, baseline, dashed peak line, and the trailing + * peak label — the single source of truth for how these charts look. Callers supply the bars and + * the tooltip content, which is where the two usages differ (stacked per-status vs. single series). + */ +export function ActivityBarChart({ + data, + max, + children, + tooltip, + peak, + peakTooltip, + width = ACTIVITY_CHART_WIDTH, +}: ActivityBarChartProps) { + return ( + <div className="flex items-start gap-1.5"> + <div className="rounded-sm" style={{ width, height: ACTIVITY_CHART_HEIGHT }}> + <BarChart + data={data as Record<string, unknown>[]} + width={width} + height={ACTIVITY_CHART_HEIGHT} + margin={{ top: 0, right: 0, left: 0, bottom: 0 }} + > + <YAxis domain={[0, max || 1]} hide /> + <Tooltip + cursor={{ fill: "rgba(255, 255, 255, 0.06)" }} + content={tooltip} + allowEscapeViewBox={{ x: true, y: true }} + wrapperStyle={{ zIndex: 1000 }} + animationDuration={0} + /> + {children} + <ReferenceLine y={0} stroke="var(--color-border-bright)" strokeWidth={1} /> + {max > 0 && ( + <ReferenceLine + y={max} + stroke="var(--color-border-brighter)" + strokeDasharray="4 4" + strokeWidth={1} + /> + )} + </BarChart> + </div> + <ActivityPeakLabel tooltip={peakTooltip}>{peak}</ActivityPeakLabel> + </div> + ); +} + +function ActivityPeakLabel({ tooltip, children }: { tooltip?: ReactNode; children: ReactNode }) { + const label = <span className={ACTIVITY_CHART_PEAK_CLASS}>{children}</span>; + if (!tooltip) return label; + return <SimpleTooltip asChild button={label} content={tooltip} />; +} diff --git a/apps/webapp/app/components/metrics/BigNumber.tsx b/apps/webapp/app/components/metrics/BigNumber.tsx index db98668e22f..5843e6229db 100644 --- a/apps/webapp/app/components/metrics/BigNumber.tsx +++ b/apps/webapp/app/components/metrics/BigNumber.tsx @@ -11,6 +11,8 @@ interface BigNumberProps { animate?: boolean; loading?: boolean; value?: number; + /** Pre-formatted display value; overrides the numeric `value` rendering when set. */ + formattedValue?: ReactNode; valueClassName?: string; defaultValue?: number; accessory?: ReactNode; @@ -22,6 +24,7 @@ interface BigNumberProps { export function BigNumber({ title, value, + formattedValue, defaultValue, valueClassName, suffix, @@ -37,7 +40,7 @@ export function BigNumber({ typeof compactThreshold === "number" && v !== undefined && v >= compactThreshold; return ( - <div className="flex flex-col justify-between gap-4 rounded-sm border border-grid-dimmed bg-background-bright p-4"> + <div className="group flex flex-col justify-between gap-4 rounded-lg border border-grid-bright bg-background-bright pb-4 pl-4 pr-3 pt-3"> <div className="flex flex-wrap items-center justify-between gap-2"> <Header3 className="leading-6">{title}</Header3> {accessory && <div className="shrink-0">{accessory}</div>} @@ -50,6 +53,11 @@ export function BigNumber({ > {loading ? ( <Spinner className="size-6" /> + ) : formattedValue !== undefined ? ( + <div className="flex flex-wrap items-baseline gap-2"> + {formattedValue} + {suffix && <div className={cn("text-xs tabular-nums", suffixClassName)}>{suffix}</div>} + </div> ) : v !== undefined ? ( <div className="flex flex-wrap items-baseline gap-2"> {shouldCompact ? ( @@ -62,7 +70,7 @@ export function BigNumber({ ) : ( formatNumber(v) )} - {suffix && <div className={cn("text-xs", suffixClassName)}>{suffix}</div>} + {suffix && <div className={cn("text-xs tabular-nums", suffixClassName)}>{suffix}</div>} </div> ) : ( "–" diff --git a/apps/webapp/app/components/metrics/MiniLineChart.tsx b/apps/webapp/app/components/metrics/MiniLineChart.tsx new file mode 100644 index 00000000000..c36e91d0b90 --- /dev/null +++ b/apps/webapp/app/components/metrics/MiniLineChart.tsx @@ -0,0 +1,222 @@ +import { type ReactNode } from "react"; +import { + Line, + LineChart, + ReferenceLine, + ResponsiveContainer, + Tooltip, + type TooltipProps, + YAxis, +} from "recharts"; +import { formatDateTime } from "~/components/primitives/DateTime"; +import { Header3 } from "~/components/primitives/Headers"; +import { SimpleTooltip } from "~/components/primitives/Tooltip"; +import TooltipPortal from "~/components/primitives/TooltipPortal"; +import { + ACTIVITY_CHART_HEIGHT, + ACTIVITY_CHART_PEAK_CLASS, + ACTIVITY_CHART_WIDTH, +} from "./ActivityBarChart"; + +type UnitLabel = { singular: string; plural: string }; + +/** Extra px above the plot so the hover activeDot at the peak value isn't clipped by the SVG edge. */ +const DOT_HEADROOM = 3; + +type MiniLineChartDatum = { + date: Date; + count: number; + /** Raw per-bucket throttled count (tooltip). */ + throttledCount: number; + /** The queued value again, present only around throttled buckets, so the warning overlay + * retraces the same line and reads as one line changing colour. */ + throttledOverlay: number | null; +}; + +export type MiniLineChartProps = { + /** Equal-width time buckets, oldest first. */ + data?: number[]; + /** + * Per-bucket throttled counts aligned 1:1 with `data`. Where throttling occurred, the queued + * line itself is retraced in the warning colour — one line that changes colour, with the + * throttled magnitude carried by the tooltip. + */ + throttled?: number[]; + /** Epoch ms of the first bucket's start. When omitted, the last bucket is anchored to now. */ + bucketStartMs?: number; + /** Width of each bucket in ms. Defaults to one hour. */ + bucketIntervalMs?: number; + /** Line colour for the queued series. */ + color?: string; + /** Trailing peak scalar shown after the chart. Defaults to the max of the buckets. */ + peak?: number; + /** Format the trailing peak label. Defaults to `toLocaleString`. */ + formatPeak?: (peak: number) => string; + /** Tooltip content shown on hover of the trailing peak label. */ + peakTooltip?: ReactNode; + /** Unit shown in the per-bucket tooltip (e.g. queued, runs). */ + unitLabel?: UnitLabel; + /** Chart width in px. Defaults to the shared ACTIVITY_CHART_WIDTH. Ignored when `fillWidth`. */ + width?: number; + /** Plot height in px. Defaults to the shared ACTIVITY_CHART_HEIGHT. */ + height?: number; + /** Stretch the plot to the container width (via ResponsiveContainer) instead of a fixed px width. */ + fillWidth?: boolean; + /** Show the trailing peak label to the right of the chart. Defaults to true. */ + showPeak?: boolean; +}; + +/** + * Inline fixed-size mini line sparkline for list rows, plus a trailing peak label. Presentational — + * the caller supplies zero/carry-forward-filled buckets. Renders an em-dash when there's no data. + * The queued series is a thin monotone line (no dots) matching the big Backlog chart; stretches + * where the queue was throttled retrace the same line in the warning colour. Shares its fixed + * dimensions and trailing peak label with {@link ActivityBarChart}, but plots lines instead of bars. + */ +export function MiniLineChart({ + data, + throttled, + bucketStartMs, + bucketIntervalMs, + color = "var(--color-tasks)", + peak: peakOverride, + formatPeak, + peakTooltip, + unitLabel = { singular: "value", plural: "values" }, + width = ACTIVITY_CHART_WIDTH, + height = ACTIVITY_CHART_HEIGHT, + fillWidth = false, + showPeak = true, +}: MiniLineChartProps) { + const hasPeakOverride = peakOverride !== undefined; + if (!data || data.length === 0 || (data.every((v) => v === 0) && !hasPeakOverride)) { + return <span className="text-text-dimmed">–</span>; + } + + // The overlay only draws where throttling happened. Mapping other buckets to null leaves gaps so + // a wholly-zero throttled series never paints over the queued line. + const hasThrottled = throttled?.some((v) => v > 0) ?? false; + + const max = Math.max(...data); + const peak = peakOverride ?? max; + + // Map each bucket to a dated point so the tooltip can show the window it represents. Buckets are + // `intervalMs` wide; if the caller didn't pass the first bucket's start, anchor the last bucket to + // now (hourly default). + const intervalMs = bucketIntervalMs ?? 3600_000; + const startMs = bucketStartMs ?? Date.now() - (data.length - 1) * intervalMs; + const chartData: MiniLineChartDatum[] = data.map((count, i) => { + const t = throttled?.[i] ?? 0; + // Extend the mask one bucket forward (a segment needs both endpoints non-null), so even a + // single throttled bucket draws a visible warning stretch. + const inOverlay = t > 0 || (throttled?.[i - 1] ?? 0) > 0; + return { + date: new Date(startMs + i * intervalMs), + count, + throttledCount: t, + throttledOverlay: inOverlay ? count : null, + }; + }); + + const chart = ( + <LineChart + data={chartData} + width={width} + height={height + DOT_HEADROOM} + margin={{ top: DOT_HEADROOM, right: 0, left: 0, bottom: 0 }} + > + <YAxis domain={[0, max || 1]} hide /> + <Tooltip + cursor={{ stroke: "rgba(255, 255, 255, 0.2)", strokeWidth: 1 }} + content={<MiniLineChartTooltip unitLabel={unitLabel} />} + allowEscapeViewBox={{ x: true, y: true }} + wrapperStyle={{ zIndex: 1000 }} + animationDuration={0} + /> + <ReferenceLine y={0} stroke="var(--color-border-bright)" strokeWidth={1} /> + <Line + type="monotone" + dataKey="count" + stroke={color} + strokeWidth={1} + dot={false} + activeDot={{ r: 2.5, fill: color, strokeWidth: 0 }} + isAnimationActive={false} + /> + {hasThrottled && ( + <Line + type="monotone" + dataKey="throttledOverlay" + stroke="var(--color-warning)" + strokeWidth={1} + dot={false} + activeDot={{ r: 2.5, fill: "var(--color-warning)", strokeWidth: 0 }} + connectNulls={false} + isAnimationActive={false} + /> + )} + </LineChart> + ); + + return ( + <div className={`flex items-start gap-1.5${fillWidth ? " w-full" : ""}`}> + {/* +DOT_HEADROOM of extra height, spent as top margin, so the hover activeDot at the peak + isn't clipped by the SVG edge while the plotted area stays `height` tall. */} + <div + className={`rounded-sm${fillWidth ? " w-full" : ""}`} + style={{ width: fillWidth ? undefined : width, height: height + DOT_HEADROOM }} + > + {/* Fixed px dims skip ResponsiveContainer's ResizeObserver (see ActivityBarChart); with + fillWidth we opt back into it so the plot stretches to the block. */} + {fillWidth ? ( + <ResponsiveContainer width="100%" height={height + DOT_HEADROOM}> + {chart} + </ResponsiveContainer> + ) : ( + chart + )} + </div> + {showPeak && ( + <MiniLinePeakLabel tooltip={peakTooltip}> + {formatPeak ? formatPeak(peak) : peak.toLocaleString()} + </MiniLinePeakLabel> + )} + </div> + ); +} + +function MiniLinePeakLabel({ tooltip, children }: { tooltip?: ReactNode; children: ReactNode }) { + const label = <span className={ACTIVITY_CHART_PEAK_CLASS}>{children}</span>; + if (!tooltip) return label; + return <SimpleTooltip asChild button={label} content={tooltip} />; +} + +function MiniLineChartTooltip({ + active, + payload, + unitLabel, +}: TooltipProps<number, string> & { unitLabel: UnitLabel }) { + if (!active || !payload || payload.length === 0) return null; + const entry = payload[0].payload as MiniLineChartDatum; + const date = entry.date instanceof Date ? entry.date : new Date(entry.date); + const formattedDate = formatDateTime(date, "UTC", [], false, true); + const throttled = entry.throttledCount; + return ( + <TooltipPortal active={active}> + <div className="rounded-sm border border-grid-bright bg-background-dimmed px-3 py-2"> + <Header3 className="border-b border-b-border-bright pb-2">{formattedDate}</Header3> + <div className="mt-2 text-xs text-text-bright"> + <span className="tabular-nums">{entry.count.toLocaleString()}</span>{" "} + <span className="text-text-dimmed"> + {entry.count === 1 ? unitLabel.singular : unitLabel.plural} + </span> + </div> + {throttled > 0 && ( + <div className="mt-1 text-xs text-warning"> + <span className="tabular-nums">{throttled.toLocaleString()}</span> throttled + </div> + )} + </div> + </TooltipPortal> + ); +} diff --git a/apps/webapp/app/components/primitives/Buttons.tsx b/apps/webapp/app/components/primitives/Buttons.tsx index 6075d52f1bf..4f4b694bccc 100644 --- a/apps/webapp/app/components/primitives/Buttons.tsx +++ b/apps/webapp/app/components/primitives/Buttons.tsx @@ -22,6 +22,15 @@ const sizes = { shortcutVariant: "small" as const, shortcut: "-ml-0.5 -mr-1.5 justify-self-center", }, + // Icon-only small button: fixed width so a row of icon buttons (with different icon + // aspect ratios) lines up, e.g. the queue block accessories. + "small-icon": { + button: "h-6 min-w-[34px] px-2 text-xs", + icon: "h-3.5 -mx-1", + iconSpacing: "gap-x-2.5", + shortcutVariant: "small" as const, + shortcut: "-ml-0.5 -mr-1.5 justify-self-center", + }, medium: { button: "h-8 px-3 text-sm", icon: "h-4 -mx-1", @@ -125,6 +134,7 @@ const variant = { "primary/large": createVariant("large", "primary"), "primary/extra-large": createVariant("extra-large", "primary"), "secondary/small": createVariant("small", "secondary"), + "secondary/small-icon": createVariant("small-icon", "secondary"), "secondary/medium": createVariant("medium", "secondary"), "secondary/large": createVariant("large", "secondary"), "secondary/extra-large": createVariant("extra-large", "secondary"), diff --git a/apps/webapp/app/components/primitives/SegmentedControl.tsx b/apps/webapp/app/components/primitives/SegmentedControl.tsx index 2f749215c29..de9a84b5c0c 100644 --- a/apps/webapp/app/components/primitives/SegmentedControl.tsx +++ b/apps/webapp/app/components/primitives/SegmentedControl.tsx @@ -76,6 +76,8 @@ type SegmentedControlProps = { variant?: VariantType; fullWidth?: boolean; onChange?: (value: string) => void; + /** Override the control's outer styling (e.g. a height that matches an adjacent input). */ + className?: string; }; export default function SegmentedControl({ @@ -86,6 +88,7 @@ export default function SegmentedControl({ variant = "secondary/medium", fullWidth, onChange, + className, }: SegmentedControlProps) { const variantStyle = variants[variant]; const _isPrimary = variant.startsWith("primary"); @@ -95,7 +98,8 @@ export default function SegmentedControl({ className={cn( "flex rounded text-text-bright", variantStyle.base, - fullWidth ? "w-full" : "w-fit" + fullWidth ? "w-full" : "w-fit", + className )} > <RadioGroup diff --git a/apps/webapp/app/components/primitives/Table.tsx b/apps/webapp/app/components/primitives/Table.tsx index 2b740fac0b8..dccb26df454 100644 --- a/apps/webapp/app/components/primitives/Table.tsx +++ b/apps/webapp/app/components/primitives/Table.tsx @@ -1,3 +1,4 @@ +import { ChevronDownIcon, ChevronUpDownIcon, ChevronUpIcon } from "@heroicons/react/20/solid"; import { ChevronRightIcon } from "@heroicons/react/24/solid"; import { Link } from "@remix-run/react"; import { ClipboardCheckIcon, ClipboardIcon } from "lucide-react"; @@ -180,7 +181,18 @@ type TableCellBasicProps = { type TableHeaderCellProps = TableCellBasicProps & { hiddenLabel?: boolean; tooltip?: ReactNode; + /** Extra class merged onto the tooltip content — e.g. widen it past the default max-width. */ + tooltipContentClassName?: string; disableTooltipHoverableContent?: boolean; + /** + * When set (together with `onSort`), the header renders a sort indicator and becomes clickable. + * `"asc"`/`"desc"` show the active direction; `null` shows the neutral (unsorted) affordance. + * This cell is presentational and fully controlled — the parent owns the sort state (see + * `useTableSort`). + */ + sortDirection?: "asc" | "desc" | null; + /** Invoked when the header is clicked or activated via keyboard. Enables sorting when provided. */ + onSort?: () => void; }; export const TableHeaderCell = forwardRef<HTMLTableCellElement, TableHeaderCellProps>( @@ -192,7 +204,10 @@ export const TableHeaderCell = forwardRef<HTMLTableCellElement, TableHeaderCellP colSpan, hiddenLabel = false, tooltip, + tooltipContentClassName, disableTooltipHoverableContent = false, + sortDirection, + onSort, }, ref ) => { @@ -207,12 +222,48 @@ export const TableHeaderCell = forwardRef<HTMLTableCellElement, TableHeaderCellP break; } - const [isHovered, setIsHovered] = useState(false); + const sortable = typeof onSort === "function"; + + const label = hiddenLabel ? <span className="sr-only">{children}</span> : children; + + const tooltipNode = tooltip ? ( + <InfoIconTooltip + content={tooltip} + contentClassName={cn("normal-case tracking-normal", tooltipContentClassName)} + disableHoverableContent={disableTooltipHoverableContent} + /> + ) : null; + + const sortIndicator = sortable ? ( + <span className="ml-1 flex items-center"> + {sortDirection === "asc" ? ( + <ChevronUpIcon className="size-4 text-text-bright" /> + ) : sortDirection === "desc" ? ( + <ChevronDownIcon className="size-4 text-text-bright" /> + ) : ( + <ChevronUpDownIcon className="size-4 text-text-dimmed transition-colors group-hover/sort:text-text-bright" /> + )} + </span> + ) : null; + + const rowClassName = cn("flex items-center gap-1", { + "justify-center": alignment === "center", + "justify-end": alignment === "right", + }); return ( <th ref={ref} scope="col" + aria-sort={ + sortable + ? sortDirection === "asc" + ? "ascending" + : sortDirection === "desc" + ? "descending" + : "none" + : undefined + } className={cn( "align-middle font-medium text-text-bright", variants[variant].headerCell, @@ -221,28 +272,29 @@ export const TableHeaderCell = forwardRef<HTMLTableCellElement, TableHeaderCellP )} colSpan={colSpan} tabIndex={-1} - onMouseEnter={() => setIsHovered(true)} - onMouseLeave={() => setIsHovered(false)} > - {hiddenLabel ? ( - <span className="sr-only">{children}</span> + {sortable ? ( + // Only the sort arrows toggle sorting — the label (and info tooltip) are not clickable, so + // clicking the header text does nothing. Order is always title → info icon → sort arrows. + <div className={rowClassName}> + {label} + {tooltip ? tooltipNode : null} + <button + type="button" + onClick={onSort} + aria-label="Toggle sort" + className="group/sort flex cursor-pointer select-none items-center rounded-sm focus-custom" + > + {sortIndicator} + </button> + </div> ) : tooltip ? ( - <div - className={cn("flex items-center gap-1", { - "justify-center": alignment === "center", - "justify-end": alignment === "right", - })} - > - {children} - <InfoIconTooltip - content={tooltip} - contentClassName="normal-case tracking-normal" - enabled={isHovered} - disableHoverableContent={disableTooltipHoverableContent} - /> + <div className={rowClassName}> + {label} + {tooltipNode} </div> ) : ( - children + label )} </th> ); @@ -259,6 +311,14 @@ type TableCellProps = TableCellBasicProps & { isSelected?: boolean; isTabbableCell?: boolean; children?: ReactNode; + /** + * Content rendered beside the cell's link/button but OUTSIDE it, so interactive adornments + * (tooltip triggers, badges that are themselves buttons) don't nest inside the `<a>`/`<button>` + * — invalid DOM that fails a11y audits. Use for a `to`/`onClick` cell that also shows a tooltip. + * `leadingContent` renders before the link, `trailingContent` after. + */ + leadingContent?: ReactNode; + trailingContent?: ReactNode; style?: React.CSSProperties; }; @@ -276,6 +336,8 @@ export const TableCell = forwardRef<HTMLTableCellElement, TableCellProps>( isSticky = false, isSelected, isTabbableCell = false, + leadingContent, + trailingContent, style, }, ref @@ -324,21 +386,71 @@ export const TableCell = forwardRef<HTMLTableCellElement, TableCellProps>( style={style} > {to ? ( - <Link - to={to} - className={cn("cursor-pointer focus:outline-hidden", flexClasses, actionClassName)} - tabIndex={isTabbableCell ? 0 : -1} - > - {children} - </Link> + // With leading/trailing content, the link is content-sized and the adornments sit beside + // it (still inside the td) so interactive triggers never nest inside the <a>. + leadingContent || trailingContent ? ( + // Stretched link: the <a> covers the whole cell via an inset ::before overlay, so the + // entire cell is clickable — not just the text. The interactive adornments (tooltip + // icons, badge buttons) sit above the overlay (relative z-10) and stay clickable, and + // never nest inside the <a>. + <div className={cn(flexClasses, "relative gap-2")}> + {leadingContent ? ( + <span className="relative z-10 flex items-center">{leadingContent}</span> + ) : null} + <Link + to={to} + className={cn( + "inline-flex items-center gap-2 before:absolute before:inset-0 before:content-[''] focus:outline-hidden", + actionClassName + )} + tabIndex={isTabbableCell ? 0 : -1} + > + {children} + </Link> + {trailingContent ? ( + <span className="relative z-10 flex items-center">{trailingContent}</span> + ) : null} + </div> + ) : ( + <Link + to={to} + className={cn("cursor-pointer focus:outline-hidden", flexClasses, actionClassName)} + tabIndex={isTabbableCell ? 0 : -1} + > + {children} + </Link> + ) ) : onClick ? ( - <button - onClick={onClick} - className={cn("cursor-pointer focus:outline-hidden", flexClasses, actionClassName)} - tabIndex={isTabbableCell ? 0 : -1} - > + leadingContent || trailingContent ? ( + <div className={cn(flexClasses, "gap-2")}> + {leadingContent} + <button + onClick={onClick} + className={cn( + "inline-flex cursor-pointer items-center gap-2 focus:outline-hidden", + actionClassName + )} + tabIndex={isTabbableCell ? 0 : -1} + > + {children} + </button> + {trailingContent} + </div> + ) : ( + <button + onClick={onClick} + className={cn("cursor-pointer focus:outline-hidden", flexClasses, actionClassName)} + tabIndex={isTabbableCell ? 0 : -1} + > + {children} + </button> + ) + ) : leadingContent || trailingContent ? ( + <div className={cn(flexClasses, "gap-2")}> + {leadingContent} {children} - </button> + {trailingContent} + </div> ) : ( <>{children}</> )} diff --git a/apps/webapp/app/components/primitives/Tooltip.tsx b/apps/webapp/app/components/primitives/Tooltip.tsx index cb9eaf0364d..6f12e408c7f 100644 --- a/apps/webapp/app/components/primitives/Tooltip.tsx +++ b/apps/webapp/app/components/primitives/Tooltip.tsx @@ -42,7 +42,7 @@ const TooltipContent = React.forwardRef< ref={ref} sideOffset={sideOffset} className={cn( - "z-50 overflow-hidden animate-in data-[side=bottom]:slide-in-from-top-1 data-[side=left]:slide-in-from-right-1 data-[side=right]:slide-in-from-left-1 data-[side=top]:slide-in-from-bottom-1 focus-visible:outline-hidden", + "z-50 max-w-[230px] overflow-hidden animate-in data-[side=bottom]:slide-in-from-top-1 data-[side=left]:slide-in-from-right-1 data-[side=right]:slide-in-from-left-1 data-[side=top]:slide-in-from-bottom-1 focus-visible:outline-hidden", variantClasses[variant], className )} diff --git a/apps/webapp/app/components/primitives/TooltipPortal.tsx b/apps/webapp/app/components/primitives/TooltipPortal.tsx index e389a9e5e72..011c5aae7ef 100644 --- a/apps/webapp/app/components/primitives/TooltipPortal.tsx +++ b/apps/webapp/app/components/primitives/TooltipPortal.tsx @@ -9,6 +9,22 @@ import useLazyRef from "~/hooks/useLazyRef"; // Recharts 3.x will have portal support, but until then we're using this: //https://github.com/recharts/recharts/issues/2458#issuecomment-1063463873 +// A portal only mounts once its tooltip is active, so its own mousemove listener attaches too late +// to know where the cursor already is — the tooltip would sit at {0,0} (top-left of the page) until +// the next mouse movement. Track the pointer globally so a newly-activated tooltip can seed its +// position immediately. +const lastPointer = { x: 0, y: 0 }; +if (typeof window !== "undefined") { + window.addEventListener( + "mousemove", + (e) => { + lastPointer.x = e.clientX; + lastPointer.y = e.clientY; + }, + { passive: true } + ); +} + export interface PopperPortalProps { active?: boolean; children: ReactNode; @@ -40,8 +56,11 @@ export default function TooltipPortal({ active = true, children }: PopperPortalP useEffect(() => { if (!active) return; + // Seed from the last known pointer so the tooltip appears at the cursor immediately, even if the + // mouse is held still after hovering onto a point (otherwise it flashes in the top-left corner). + virtualElementRef.current?.update(lastPointer.x, lastPointer.y); update?.(); - }, [active, update]); + }, [active, update, virtualElementRef]); if (!portalElement) return null; @@ -53,6 +72,9 @@ export default function TooltipPortal({ active = true, children }: PopperPortalP ...styles.popper, zIndex: 1000, display: active ? "block" : "none", + // The tooltip sits just under the cursor; without this, moving along the line drags the + // cursor onto the tooltip, which fires the chart's mouseleave and flickers it off/on. + pointerEvents: "none", }} > {children} diff --git a/apps/webapp/app/components/primitives/UsageSparkline.tsx b/apps/webapp/app/components/primitives/UsageSparkline.tsx index 5a6b0a5c12d..89c471a405e 100644 --- a/apps/webapp/app/components/primitives/UsageSparkline.tsx +++ b/apps/webapp/app/components/primitives/UsageSparkline.tsx @@ -27,10 +27,16 @@ export type UsageSparklineProps = { color?: string; /** Unit shown in the tooltip (e.g. calls, tokens). */ unitLabel?: UnitLabel; + /** Trailing scalar shown after the chart. Defaults to the sum of buckets (override for gauges, e.g. peak). */ + total?: number; /** Format the trailing total. Defaults to `toLocaleString`. */ formatTotal?: (total: number) => string; /** Class for the trailing total label. */ totalClassName?: string; + /** Size of the bar chart. Defaults to the list-cell size (`h-6 w-28`). */ + chartClassName?: string; + /** Hide the trailing total (e.g. when the caller shows its own headline). */ + hideTotal?: boolean; }; /** @@ -44,14 +50,18 @@ export function UsageSparkline({ bucketIntervalMs, color = "var(--color-pending)", unitLabel = { singular: "call", plural: "calls" }, + total: totalOverride, formatTotal, totalClassName = "text-blue-400", + chartClassName = "h-6 w-28", + hideTotal = false, }: UsageSparklineProps) { - if (!data || data.every((v) => v === 0)) { + const hasTotalOverride = totalOverride !== undefined; + if (!data || data.length === 0 || (data.every((v) => v === 0) && !hasTotalOverride)) { return <span className="text-text-dimmed">–</span>; } - const total = data.reduce((a, b) => a + b, 0); + const total = totalOverride ?? data.reduce((a, b) => a + b, 0); const max = Math.max(...data); // Map each bucket to a dated point so the tooltip can show the window it @@ -66,7 +76,7 @@ export function UsageSparkline({ return ( <div className="flex items-start gap-2"> - <div className="h-6 w-28 rounded-sm"> + <div className={cn("rounded-sm", chartClassName)}> <ResponsiveContainer width="100%" height="100%"> <BarChart data={chartData} margin={{ top: 0, right: 0, left: 0, bottom: 0 }}> <YAxis domain={[0, max || 1]} hide /> @@ -96,9 +106,11 @@ export function UsageSparkline({ </BarChart> </ResponsiveContainer> </div> - <span className={cn("-mt-1 text-xs tabular-nums", totalClassName)}> - {formatTotal ? formatTotal(total) : total.toLocaleString()} - </span> + {hideTotal ? null : ( + <span className={cn("-mt-1 text-xs tabular-nums", totalClassName)}> + {formatTotal ? formatTotal(total) : total.toLocaleString()} + </span> + )} </div> ); } diff --git a/apps/webapp/app/components/primitives/charts/Card.tsx b/apps/webapp/app/components/primitives/charts/Card.tsx index f4fb13b51bc..d197b109b01 100644 --- a/apps/webapp/app/components/primitives/charts/Card.tsx +++ b/apps/webapp/app/components/primitives/charts/Card.tsx @@ -19,7 +19,7 @@ const CardHeader = ({ children, draggable }: { children: ReactNode; draggable?: return ( <Header3 className={cn( - "drag-handle mb-3 flex items-center justify-between gap-2 pl-4 pr-3", + "drag-handle mb-3 flex items-start justify-between gap-2 pl-4 pr-3", draggable && "cursor-grab active:cursor-grabbing" )} > diff --git a/apps/webapp/app/components/primitives/charts/Chart.tsx b/apps/webapp/app/components/primitives/charts/Chart.tsx index 85d28336837..932eaa516b1 100644 --- a/apps/webapp/app/components/primitives/charts/Chart.tsx +++ b/apps/webapp/app/components/primitives/charts/Chart.tsx @@ -176,7 +176,10 @@ const ChartTooltipContent = React.forwardRef< {payload.map((item, index) => { const key = `${nameKey || item.name || item.dataKey || "value"}`; const itemConfig = getPayloadConfigFromPayload(config, item, key); - const indicatorColor = color || item.payload.fill || item.color; + // Prefer the series' configured colour over item.color: a threshold/gradient line's + // recharts colour is a `url(#…)` gradient ref, which is invalid as a CSS background and + // renders no swatch. The config colour is the intended solid series colour. + const indicatorColor = color || itemConfig?.color || item.payload.fill || item.color; return ( <div @@ -216,7 +219,7 @@ const ChartTooltipContent = React.forwardRef< )} <div className={cn( - "flex flex-1 justify-between leading-none", + "flex flex-1 justify-between gap-3 leading-none", nestLabel ? "items-end" : "items-center" )} > diff --git a/apps/webapp/app/components/primitives/charts/ChartBar.tsx b/apps/webapp/app/components/primitives/charts/ChartBar.tsx index 73ffb35c588..8f18ff7e13a 100644 --- a/apps/webapp/app/components/primitives/charts/ChartBar.tsx +++ b/apps/webapp/app/components/primitives/charts/ChartBar.tsx @@ -260,7 +260,7 @@ export function ChartBarRenderer({ {/* When legend is shown below the chart, render tooltip with cursor only (no content popup). Otherwise render the full tooltip with zoom instructions. */} <ChartTooltip - cursor={{ fill: "rgba(255, 255, 255, 0.06)" }} + cursor={{ fill: "rgba(255, 255, 255, 0.12)" }} content={ syncZoomSelection && zoomFrom != null && zoomTo != null ? ( <ZoomRangeTooltip from={zoomFrom} to={zoomTo} /> diff --git a/apps/webapp/app/components/primitives/charts/ChartCard.tsx b/apps/webapp/app/components/primitives/charts/ChartCard.tsx index 367bb703ea4..69464cf3406 100644 --- a/apps/webapp/app/components/primitives/charts/ChartCard.tsx +++ b/apps/webapp/app/components/primitives/charts/ChartCard.tsx @@ -7,6 +7,7 @@ import { useShortcutKeys } from "~/hooks/useShortcutKeys"; import { cn } from "~/utils/cn"; import { Dialog, DialogContent, DialogHeader } from "../Dialog"; import { Card } from "./Card"; +import { ChartSyncProvider, useChartSync } from "./ChartSyncContext"; type ChartCardProps = { /** Title shown in the card header (and the fullscreen dialog header). */ @@ -34,6 +35,10 @@ export function ChartCard({ }: ChartCardProps) { const [isFullscreen, setIsFullscreen] = useState(false); const containerRef = useRef<HTMLDivElement>(null); + // A maximized chart is its own sync group: hover + drag-select shouldn't mirror onto the + // (hidden) sibling charts behind the dialog. Give it a fresh provider with isolated state, + // but inherit the page group's onZoom so drag-to-zoom still sets the time filter. + const parentSync = useChartSync(); // "v" toggles fullscreen for the hovered card. useShortcutKeys({ @@ -83,9 +88,17 @@ export function ChartCard({ {maximizable && ( <Dialog open={isFullscreen} onOpenChange={setIsFullscreen}> <DialogContent fullscreen className="flex flex-col bg-background-bright"> - <DialogHeader>{title}</DialogHeader> + {/* In fullscreen, space the title's legend (the flex-col title node) further from the + title — gap-6 instead of the card's gap-1. */} + <DialogHeader className="[&>span]:gap-6">{title}</DialogHeader> <div className="min-h-0 w-full flex-1 overflow-hidden pt-4"> - {fullscreenChildren ?? children} + {parentSync ? ( + <ChartSyncProvider onZoom={parentSync.onZoom}> + {fullscreenChildren ?? children} + </ChartSyncProvider> + ) : ( + (fullscreenChildren ?? children) + )} </div> </DialogContent> </Dialog> diff --git a/apps/webapp/app/components/primitives/charts/ChartLine.tsx b/apps/webapp/app/components/primitives/charts/ChartLine.tsx index 58229bc5e6a..000d74b91c0 100644 --- a/apps/webapp/app/components/primitives/charts/ChartLine.tsx +++ b/apps/webapp/app/components/primitives/charts/ChartLine.tsx @@ -1,22 +1,35 @@ +import { useId } from "react"; import { Area, AreaChart, CartesianGrid, Line, LineChart, + ReferenceArea, + ReferenceLine, XAxis, YAxis, type XAxisProps, type YAxisProps, } from "recharts"; import { ChartTooltip, ChartTooltipContent } from "~/components/primitives/charts/Chart"; +import TooltipPortal from "~/components/primitives/TooltipPortal"; import { CHART_MARGIN } from "./ChartBar"; import { useChartContext } from "./ChartContext"; import { ChartLineInvalid, ChartLineLoading, ChartLineNoData } from "./ChartLoading"; import { useHasNoData } from "./ChartRoot"; +import { useChartSync } from "./ChartSyncContext"; import { defaultYAxisTickFormatter, useYAxisWidth } from "./useYAxisWidth"; // Legend is now rendered by ChartRoot outside the chart container +// Dashed line mirroring the hovered x across synced charts. +const SYNC_LINE_COLOR = "var(--color-text-faint)"; + +// Data key the warning overlay line is plotted under. Injected into the render data only; never +// added to config/series, so it stays out of the legend, no-data check and series totals. Deduped +// out of the tooltip so a hovered bucket shows one value, not the base + overlay retrace. +const WARNING_OVERLAY_KEY = "__warningOverlay"; + type CurveType = | "basis" | "basisClosed" @@ -31,6 +44,42 @@ type CurveType = | "stepBefore" | "stepAfter"; +/** While drag-to-zooming, show the selected From/To range instead of hovered values. */ +function ZoomRangeTooltip({ active, from, to }: { active?: boolean; from: string; to: string }) { + if (!active) return null; + return ( + <TooltipPortal active={active}> + <div className="grid grid-cols-[auto_auto] gap-x-2 gap-y-1 rounded-lg border border-grid-bright bg-background-bright px-2.5 py-1.5 text-xs shadow-xl"> + <span className="text-right text-text-dimmed">From:</span> + <span className="tabular-nums text-text-bright">{from}</span> + <span className="text-right text-text-dimmed">To:</span> + <span className="tabular-nums text-text-bright">{to}</span> + </div> + </TooltipPortal> + ); +} + +// Stable module-level tooltip for warning-overlay charts: drops the overlay's retraced entry so a +// hovered bucket shows one value (base), not base + overlay. Defined at module scope (not inline in +// the renderer) so recharts reconciles it in place across hover re-renders instead of remounting +// the portaled tooltip — the latter caused a flicker while moving along the line. +function OverlayFilteredTooltip(props: any) { + return ( + <ChartTooltipContent + {...props} + payload={props.payload?.filter((p: any) => p.dataKey !== WARNING_OVERLAY_KEY)} + /> + ); +} + +// Stable module-level tooltip for the stacked area chart: keeps the line-style indicator the +// stacked view has always used (ChartTooltipContent otherwise defaults to a dot). Module-level for +// the same reconcile-in-place reason as OverlayFilteredTooltip — an inline element would remount +// the portaled tooltip on every hover re-render and flicker. +function StackedAreaTooltip(props: any) { + return <ChartTooltipContent {...props} indicator="line" />; +} + // ============================================================================ // COMPOUND COMPONENT API // ============================================================================ @@ -48,16 +97,116 @@ export type ChartLineRendererProps = { tooltipLabelFormatter?: (label: string, payload: any[]) => string; /** Optional formatter for numeric tooltip values (e.g. bytes, duration) */ tooltipValueFormatter?: (value: number) => string; + /** Draw a dot at each data point. Defaults to true; turn off for dense/compact charts. */ + showDots?: boolean; + /** + * Horizontal reference lines (e.g. limits); the y-domain extends to include them. + * + * `labelPlacement` controls where the label sits relative to the plot: + * - `"inside"` (default): right-aligned just below the line, inside the plot area. + * - `"outside"`: in the right gutter at the line's y. The chart's right margin is widened + * automatically so outside labels are not clipped by the SVG viewport. + */ + referenceLines?: Array<{ + y: number; + label?: string; + color?: string; + labelPlacement?: "inside" | "outside"; + }>; + /** + * Recolor the stroke above a threshold value (e.g. an over-limit warning). The y-domain is + * pinned so the gradient split lines up exactly with the plotted values and reference lines. + * Single-series (non-stacked) line charts only. + * + * The gradient offset is derived from the plotted line's own value range (objectBoundingBox maps + * 0..1 to the path's bounding box, not the y-axis), so the colour change lands exactly at the + * threshold value however the domain is padded for reference lines. `series` targets which line + * the gradient applies to (others keep their own colour); defaults to the first series. + */ + thresholdStroke?: { value: number; aboveColor: string; series?: string }; + /** + * Per-bucket warning recolour: a series is retraced in the warning colour only across buckets + * where it crosses a limit — either strictly above a constant `threshold` (single-series case, + * applied to the first series), or where one series drops below another (`series` below `below`, + * e.g. started < enqueued = "not keeping up"). The mask extends one bucket forward so a lone + * crossing still yields a visible segment. Unlike {@link thresholdStroke}'s gradient split, + * non-crossing buckets always stay the base colour. The overlay is excluded from the legend and + * deduped out of the tooltip. Non-stacked line charts only. + */ + warningOverlay?: + | { threshold: number; color?: string } + | { series: string; below: string; color?: string } + | { series: string; atOrAbove: string; color?: string }; /** Width injected by ResponsiveContainer */ width?: number; /** Height injected by ResponsiveContainer */ height?: number; }; +/** Font size used for reference-line labels; also used to size the outside-label gutter. */ +const REFERENCE_LABEL_FONT_SIZE = 10; + +/** + * Reference-line label (recharts injects viewBox). + * - `"inside"`: right-aligned just below the line, inside the plot. + * - `"outside"`: left-aligned in the right gutter, vertically centered on the line. + */ +function ReferenceLineLabel({ + viewBox, + value, + placement = "inside", +}: { + viewBox?: { x: number; y: number; width: number }; + value: string; + placement?: "inside" | "outside"; +}) { + if (!viewBox) return null; + if (placement === "outside") { + return ( + <text + x={viewBox.x + viewBox.width + 6} + y={viewBox.y} + dominantBaseline="middle" + textAnchor="start" + fill="#878C99" + fontSize={REFERENCE_LABEL_FONT_SIZE} + > + {value} + </text> + ); + } + return ( + <text + x={viewBox.x + viewBox.width - 4} + y={viewBox.y + 12} + textAnchor="end" + fill="#878C99" + fontSize={REFERENCE_LABEL_FONT_SIZE} + > + {value} + </text> + ); +} + +/** + * Extra right margin (px) needed so outside reference-line labels aren't clipped by the SVG + * viewport. Estimates label width from character count; returns 0 when no label is outside-placed. + */ +function outsideLabelGutter(referenceLines: ChartLineRendererProps["referenceLines"]): number { + const outside = (referenceLines ?? []).filter((l) => l.labelPlacement === "outside" && l.label); + if (outside.length === 0) return 0; + const maxChars = Math.max(...outside.map((l) => l.label!.length)); + // ~0.62em per char at this font size, plus padding on both sides of the label. + return Math.ceil(maxChars * REFERENCE_LABEL_FONT_SIZE * 0.62) + 12; +} + /** * Line chart renderer for the compound component system. * Must be used within a Chart.Root. * + * When wrapped in a <ChartSyncProvider>, participates in the group's shared hover + * indicator and drag-to-zoom (mirrors Chart.Bar; a no-op when no provider is present). + * * @example * ```tsx * <Chart.Root config={config} data={data} dataKey="day"> @@ -73,6 +222,10 @@ export function ChartLineRenderer({ stacked = false, tooltipLabelFormatter, tooltipValueFormatter, + showDots = true, + referenceLines, + thresholdStroke, + warningOverlay, width, height, }: ChartLineRendererProps) { @@ -88,6 +241,9 @@ export function ChartLineRenderer({ showLegend, } = useChartContext(); const hasNoData = useHasNoData(); + const sync = useChartSync(); + // Strip the colons React injects (":r1:") so the id is safe inside an SVG url(#…) reference. + const gradientId = `line-threshold-${useId().replace(/:/g, "")}`; const yAxisTickFormatter = yAxisPropsProp?.tickFormatter ?? defaultYAxisTickFormatter; const computedYAxisWidth = useYAxisWidth(data, visibleSeries, yAxisTickFormatter); @@ -100,18 +256,13 @@ export function ChartLineRenderer({ return <ChartLineInvalid />; } - // Get the x-axis ticks based on tooltip state - const xAxisTicks = - highlight.tooltipActive && data.length > 2 - ? [data[0]?.[dataKey], data[data.length - 1]?.[dataKey]] - : undefined; - const xAxisConfig = { dataKey, tickLine: false, axisLine: false, tickMargin: 10, - ticks: xAxisTicks, + // Keep every x-axis label visible at all times, including on hover. Previously the axis + // collapsed to just the first + last tick while the tooltip was active. interval: "preserveStartEnd" as const, tick: { fill: "var(--color-text-dimmed)", @@ -121,6 +272,75 @@ export function ChartLineRenderer({ ...xAxisPropsProp, }; + // A threshold stroke needs an exact, fixed y-domain so the gradient split aligns with the + // plotted values and the reference lines. Compute it from the data + reference/threshold ys. + let thresholdActive = false; + let thresholdOffset = 0; + if (thresholdStroke && !stacked) { + thresholdActive = true; + // The gradient is objectBoundingBox — its 0..1 maps to the plotted line's own bounding box + // (lineMax at the top, lineMin at the bottom), NOT the y-axis. So derive the split from the + // target line's value range: offset = (lineMax - threshold) / (lineMax - lineMin) lands the + // colour change exactly at the threshold value's pixel, whatever the axis domain is. That means + // we don't pin the domain (which coarsened the ticks) — it auto-scales as usual. + const gradientKey = thresholdStroke.series ?? visibleSeries[0]; + let lineMin = Infinity; + let lineMax = -Infinity; + for (const row of data) { + const v = Number(row[gradientKey]); + if (Number.isFinite(v)) { + if (v < lineMin) lineMin = v; + if (v > lineMax) lineMax = v; + } + } + if (!Number.isFinite(lineMin)) { + lineMin = 0; + lineMax = thresholdStroke.value; + } + const range = lineMax - lineMin; + thresholdOffset = + range > 0 + ? Math.min(1, Math.max(0, (lineMax - thresholdStroke.value) / range)) + : lineMax >= thresholdStroke.value + ? 0 + : 1; + } + + // Per-bucket warning overlay: single-series line charts only. Retrace the primary series in the + // warning colour, non-null only across over-threshold stretches. Include the immediate neighbours + // of an over-threshold bucket (both endpoints of a segment must be non-null), so the crossing + // segment on BOTH sides is drawn — the colour change tracks the axis crossing symmetrically, and + // a lone over-threshold bucket still yields a visible segment. + const overlayKey = + warningOverlay && !stacked + ? "series" in warningOverlay + ? warningOverlay.series + : visibleSeries[0] + : undefined; + const overlayActive = overlayKey != null; + const chartData = overlayActive + ? data.map((row, i) => { + const isOver = (r: (typeof data)[number] | undefined) => { + if (!r) return false; + const v = Number(r[overlayKey]); + if (!Number.isFinite(v)) return false; + if ("below" in warningOverlay!) { + // "Not keeping up": the series dips below its companion (e.g. started < enqueued). + const b = Number(r[warningOverlay.below]); + return Number.isFinite(b) && v < b; + } + if ("atOrAbove" in warningOverlay!) { + // "At the limit": the series reaches or exceeds its companion (e.g. running >= limit). + const b = Number(r[warningOverlay.atOrAbove]); + return Number.isFinite(b) && b > 0 && v >= b; + } + return v > warningOverlay!.threshold; + }; + const inOverlay = isOver(data[i - 1]) || isOver(row) || isOver(data[i + 1]); + return { ...row, [WARNING_OVERLAY_KEY]: inOverlay ? row[overlayKey] : null }; + }) + : data; + const yAxisConfig = { axisLine: false, tickLine: false, @@ -135,47 +355,160 @@ export function ChartLineRenderer({ ...yAxisPropsProp, }; - // Handle mouse leave to also reset highlight + // Widen the right margin only when a reference line is outside-labeled, so charts without + // outside labels keep their existing geometry. + const rightGutter = outsideLabelGutter(referenceLines); + const chartMargin = + rightGutter > 0 + ? { ...CHART_MARGIN, right: Math.max(CHART_MARGIN.right, rightGutter) } + : CHART_MARGIN; + + // Handle mouse leave to also reset highlight and any synced hover/zoom drag. const handleMouseLeave = () => { highlight.setTooltipActive(false); highlight.reset(); + sync?.setActiveX(null); + sync?.cancelZoom(); + }; + + // Synced hover + drag-to-zoom state (mirrors Chart.Bar; all no-ops without a provider). + const syncActiveX = sync?.activeX ?? null; + const syncZoomSelection = sync?.zoomSelection ?? null; + const bucketWidthMs = data.length >= 2 ? Number(data[1][dataKey]) - Number(data[0][dataKey]) : 0; + const formatZoomEdge = (v: number): string => + tooltipLabelFormatter ? tooltipLabelFormatter("", [{ payload: { [dataKey]: v } }]) : String(v); + let zoomFrom: string | null = null; + let zoomTo: string | null = null; + if (syncZoomSelection) { + const a = Number(syncZoomSelection.start); + const b = Number(syncZoomSelection.current); + if (Number.isFinite(a) && Number.isFinite(b)) { + zoomFrom = formatZoomEdge(Math.min(a, b)); + zoomTo = formatZoomEdge(Math.max(a, b)); + } + } + + const sharedMouseHandlers = { + className: sync?.zoomEnabled ? "cursor-crosshair select-none" : undefined, + onMouseDown: (e: any) => { + if (sync?.zoomEnabled && e?.activeLabel != null) sync.startZoom(e.activeLabel); + }, + onMouseMove: (e: any) => { + if (sync?.zoomEnabled && sync.zoomSelection && e?.activeLabel != null) { + sync.updateZoom(e.activeLabel); + } + if (e?.activePayload?.length) { + setActivePayload(e.activePayload, e.activeTooltipIndex); + highlight.setTooltipActive(true); + sync?.setActiveX(e.activeLabel ?? null); + } else { + highlight.setTooltipActive(false); + sync?.setActiveX(null); + } + }, + onMouseUp: () => { + if (sync?.zoomEnabled) sync.endZoom(bucketWidthMs); + }, + onMouseLeave: handleMouseLeave, }; + // Pass the tooltip as a stable ELEMENT (not an inline function). recharts remounts a function + // `content` on the re-renders that fire while hovering along the line (sync/highlight state + // updates), which unmounts the portaled tooltip every bucket = flicker. An element of a + // module-level component type reconciles in place, so the tooltip stays mounted while moving. + const tooltipContent = + syncZoomSelection && zoomFrom != null && zoomTo != null ? ( + <ZoomRangeTooltip from={zoomFrom} to={zoomTo} /> + ) : showLegend ? ( + () => null + ) : overlayActive ? ( + <OverlayFilteredTooltip valueFormatter={tooltipValueFormatter} /> + ) : ( + <ChartTooltipContent valueFormatter={tooltipValueFormatter} /> + ); + + const referenceOverlays = ( + <> + {/* Synced drag-to-zoom selection — mirrored across charts in the same group. */} + {syncZoomSelection && ( + <ReferenceArea + x1={syncZoomSelection.start} + x2={syncZoomSelection.current} + isFront + stroke="var(--color-pending)" + strokeOpacity={0.3} + fill="var(--color-pending)" + fillOpacity={0.15} + className="pointer-events-none" + /> + )} + {/* Synced hover indicator: drawn on the *other* charts only (the hovered one shows its + own cursor); pointer-events-none so it never steals hover. */} + {syncActiveX != null && !highlight.tooltipActive && ( + <ReferenceLine + x={syncActiveX} + stroke={SYNC_LINE_COLOR} + strokeWidth={1} + strokeDasharray="4 4" + isFront + className="pointer-events-none" + /> + )} + {referenceLines?.map((line) => ( + <ReferenceLine + key={`ref-${line.y}-${line.label ?? ""}`} + y={line.y} + stroke={line.color ?? "#4D525B"} + strokeDasharray="4 4" + strokeWidth={1} + // extendDomain (not "hidden") means recharts does NOT wrap this layer in the plot + // clipPath, so an outside-placed label can render into the right gutter unclipped. + ifOverflow="extendDomain" + label={ + line.label ? ( + <ReferenceLineLabel value={line.label} placement={line.labelPlacement} /> + ) : undefined + } + /> + ))} + </> + ); + // Render stacked area chart if stacked prop is true if (stacked && visibleSeries.length > 1) { + // Same variants as the line chart's tooltipContent, but the default popup keeps the stacked + // view's line-style indicator (warning overlay never applies to stacked areas). + const stackedTooltipContent = + syncZoomSelection && zoomFrom != null && zoomTo != null ? ( + <ZoomRangeTooltip from={zoomFrom} to={zoomTo} /> + ) : showLegend ? ( + () => null + ) : ( + <StackedAreaTooltip valueFormatter={tooltipValueFormatter} /> + ); return ( <AreaChart data={data} width={width} height={height} stackOffset="none" - margin={CHART_MARGIN} - onMouseMove={(e: any) => { - if (e?.activePayload?.length) { - setActivePayload(e.activePayload, e.activeTooltipIndex); - highlight.setTooltipActive(true); - } else { - highlight.setTooltipActive(false); - } - }} - onMouseLeave={handleMouseLeave} + margin={chartMargin} + {...sharedMouseHandlers} > <CartesianGrid vertical={false} stroke="var(--color-grid-bright)" strokeDasharray="3 3" /> <XAxis {...xAxisConfig} /> <YAxis {...yAxisConfig} /> {/* When legend is shown below, render tooltip with cursor only (no content popup) */} <ChartTooltip - cursor={{ stroke: "rgba(255, 255, 255, 0.1)", strokeWidth: 1 }} - content={ - showLegend ? ( - () => null - ) : ( - <ChartTooltipContent indicator="line" valueFormatter={tooltipValueFormatter} /> - ) - } + cursor={{ stroke: SYNC_LINE_COLOR, strokeWidth: 1 }} + content={stackedTooltipContent} labelFormatter={tooltipLabelFormatter} + isAnimationActive={false} + allowEscapeViewBox={{ x: true, y: true }} + wrapperStyle={{ zIndex: 1000 }} /> {/* Note: Legend is now rendered by ChartRoot outside the chart container */} + {referenceOverlays} {visibleSeries.map((key) => ( <Area key={key} @@ -196,44 +529,130 @@ export function ChartLineRenderer({ return ( <LineChart accessibilityLayer - data={data} + data={chartData} width={width} height={height} - margin={CHART_MARGIN} - onMouseMove={(e: any) => { - if (e?.activePayload?.length) { - setActivePayload(e.activePayload, e.activeTooltipIndex); - highlight.setTooltipActive(true); - } else { - highlight.setTooltipActive(false); - } - }} - onMouseLeave={handleMouseLeave} + margin={chartMargin} + {...sharedMouseHandlers} > + {thresholdActive ? ( + <defs> + <linearGradient id={gradientId} x1="0" y1="0" x2="0" y2="1"> + <stop offset={thresholdOffset} stopColor={thresholdStroke!.aboveColor} /> + <stop + offset={thresholdOffset} + stopColor={config[thresholdStroke!.series ?? visibleSeries[0]]?.color} + /> + </linearGradient> + </defs> + ) : null} <CartesianGrid vertical={false} stroke="var(--color-grid-bright)" strokeDasharray="3 3" /> <XAxis {...xAxisConfig} /> <YAxis {...yAxisConfig} /> {/* When legend is shown below, render tooltip with cursor only (no content popup) */} <ChartTooltip - cursor={{ stroke: "rgba(255, 255, 255, 0.1)", strokeWidth: 1 }} - content={ - showLegend ? () => null : <ChartTooltipContent valueFormatter={tooltipValueFormatter} /> - } + cursor={{ stroke: SYNC_LINE_COLOR, strokeWidth: 1 }} + content={tooltipContent} labelFormatter={tooltipLabelFormatter} + isAnimationActive={false} + allowEscapeViewBox={{ x: true, y: true }} + wrapperStyle={{ zIndex: 1000 }} /> {/* Note: Legend is now rendered by ChartRoot outside the chart container */} - {visibleSeries.map((key) => ( + {referenceOverlays} + {visibleSeries.map((key) => { + // The gradient stroke only applies to the threshold's target series (default: the first); + // other series (e.g. the grey limit line) keep their own colour. + const gradientLine = + thresholdActive && (thresholdStroke!.series == null || thresholdStroke!.series === key); + return ( + <Line + key={key} + dataKey={key} + type={lineType} + stroke={gradientLine ? `url(#${gradientId})` : config[key]?.color} + strokeWidth={1} + dot={showDots ? { r: 1.5, fill: config[key]?.color, strokeWidth: 0 } : false} + // The hover dot matches the line colour under it: for a gradient (threshold) line it + // flips at the split; otherwise it's the series colour. The warning overlay draws its + // own dot on top where it's active. + activeDot={ + gradientLine + ? (props: ActiveDotProps) => ( + <ThresholdActiveDot + {...props} + dataKey={key} + threshold={thresholdStroke!.value} + aboveColor={thresholdStroke!.aboveColor} + baseColor={config[key]?.color ?? "var(--color-tasks)"} + /> + ) + : { r: 4, fill: config[key]?.color, strokeWidth: 0 } + } + isAnimationActive={false} + /> + ); + })} + {overlayActive && ( + // Drawn after the base line so the warning colour sits on top. connectNulls={false} keeps + // the mask to over-threshold stretches; excluded from the legend and (above) the tooltip. + // Its active dot inherits the warning colour so the hover dot is yellow over yellow. + // Same 1px width as the base line so the warning stretch matches the other lines — it traces + // the same points over its over-threshold buckets, so it covers the base exactly. <Line - key={key} - dataKey={key} + key={WARNING_OVERLAY_KEY} + dataKey={WARNING_OVERLAY_KEY} type={lineType} - stroke={config[key]?.color} + stroke={warningOverlay!.color ?? "var(--color-warning)"} strokeWidth={1} - dot={{ r: 1.5, fill: config[key]?.color, strokeWidth: 0 }} - activeDot={{ r: 4 }} + dot={false} + activeDot={{ + r: 4, + fill: warningOverlay!.color ?? "var(--color-warning)", + strokeWidth: 0, + }} + connectNulls={false} + legendType="none" isAnimationActive={false} /> - ))} + )} </LineChart> ); } + +type ActiveDotProps = { + cx?: number; + cy?: number; + value?: number | Array<number>; + payload?: Record<string, unknown>; +}; + +/** Hover dot for a gradient (threshold) line: filled with the colour of the line under it — + * the warning colour at/above the threshold, the base colour below. Reads the bucket value from + * `payload[dataKey]` (robust across recharts versions) and falls back to the `value` prop. */ +function ThresholdActiveDot({ + cx, + cy, + value, + payload, + dataKey, + threshold, + aboveColor, + baseColor, +}: ActiveDotProps & { + dataKey: string; + threshold: number; + aboveColor: string; + baseColor: string; +}) { + if (cx === undefined || cy === undefined) return null; + const fromPayload = payload?.[dataKey]; + const raw = + typeof fromPayload === "number" + ? fromPayload + : Array.isArray(value) + ? value[value.length - 1] + : value; + const color = typeof raw === "number" && raw > threshold ? aboveColor : baseColor; + return <circle cx={cx} cy={cy} r={4} fill={color} />; +} diff --git a/apps/webapp/app/components/primitives/charts/ChartSyncContext.tsx b/apps/webapp/app/components/primitives/charts/ChartSyncContext.tsx index ef01e1ded8b..6ef7f073dd0 100644 --- a/apps/webapp/app/components/primitives/charts/ChartSyncContext.tsx +++ b/apps/webapp/app/components/primitives/charts/ChartSyncContext.tsx @@ -25,6 +25,8 @@ type ChartSyncContextValue = { /** Whether drag-to-zoom is active (an onZoom handler was provided). */ zoomEnabled: boolean; + /** The commit handler, re-exposed so a nested group (e.g. a fullscreen chart) can inherit it. */ + onZoom?: (range: ChartZoomRange) => void; /** Current drag selection, mirrored across charts; null when not dragging. */ zoomSelection: ZoomSelection | null; startZoom: (x: number | string) => void; @@ -103,6 +105,7 @@ export function ChartSyncProvider({ activeX, setActiveX, zoomEnabled: onZoom != null, + onZoom, zoomSelection, startZoom, updateZoom, diff --git a/apps/webapp/app/components/primitives/useTableSort.ts b/apps/webapp/app/components/primitives/useTableSort.ts new file mode 100644 index 00000000000..5bb5f600431 --- /dev/null +++ b/apps/webapp/app/components/primitives/useTableSort.ts @@ -0,0 +1,130 @@ +import { useCallback, useMemo, useState } from "react"; + +export type SortDirection = "asc" | "desc"; + +export type SortState<K extends string = string> = { + key: K; + direction: SortDirection; +}; + +/** + * A sortable column definition for {@link useTableSort}. + * + * - `"number"`: sorts numerically. `null`/`undefined`/`NaN` values always sort last, + * regardless of the sort direction. + * - `"alpha"`: sorts with `localeCompare`, case-insensitively. Empty/nullish values sort last. + * - `"custom"`: sorts with the provided comparator `(a, b) => number` (asc order); the direction + * flip is applied on top for you. + * + * In every case the sort is stable: rows that compare equal keep their original relative order, + * and with no active sort the rows are returned untouched. + */ +export type SortColumn<T, K extends string = string> = + | { key: K; type: "number"; value: (row: T) => number | null | undefined } + | { key: K; type: "alpha"; value: (row: T) => string | null | undefined } + | { key: K; type: "custom"; compare: (a: T, b: T) => number }; + +/** Presentational props to spread onto a `<TableHeaderCell>` for a given column. */ +export type TableSortHeaderProps = { + sortDirection: SortDirection | null; + onSort: () => void; +}; + +export function compareColumn<T, K extends string>( + column: SortColumn<T, K>, + a: T, + b: T, + direction: SortDirection +): number { + const sign = direction === "asc" ? 1 : -1; + + if (column.type === "custom") { + return sign * column.compare(a, b); + } + + if (column.type === "number") { + const av = column.value(a); + const bv = column.value(b); + const aNull = av === null || av === undefined || Number.isNaN(av); + const bNull = bv === null || bv === undefined || Number.isNaN(bv); + // Nulls always sort last, independent of direction. + if (aNull && bNull) return 0; + if (aNull) return 1; + if (bNull) return -1; + return sign * (av - bv); + } + + // alpha + const av = column.value(a); + const bv = column.value(b); + const aEmpty = av === null || av === undefined || av === ""; + const bEmpty = bv === null || bv === undefined || bv === ""; + if (aEmpty && bEmpty) return 0; + if (aEmpty) return 1; + if (bEmpty) return -1; + return sign * av.localeCompare(bv, undefined, { sensitivity: "base" }); +} + +/** + * Stable sort of `rows` by a single `column`/`direction`. Rows that compare equal keep their + * original relative order. Pure and side-effect free — exported so the sort behavior can be + * unit-tested without rendering a component. + */ +export function sortRows<T, K extends string>( + rows: ReadonlyArray<T>, + column: SortColumn<T, K>, + direction: SortDirection +): T[] { + return rows + .map((row, index) => ({ row, index })) + .sort((a, b) => { + const result = compareColumn(column, a.row, b.row, direction); + return result !== 0 ? result : a.index - b.index; + }) + .map((entry) => entry.row); +} + +/** + * Client-side, header-click column sorting for tables of any row shape. + * + * Clicking a column cycles asc -> desc -> cleared (back to the original row order), so the + * incoming order (e.g. a server default) is always reachable without a reload. Returns the + * sorted rows plus a `getSortProps(key)` helper whose result spreads straight onto + * `<TableHeaderCell>`. + */ +export function useTableSort<T, K extends string = string>( + rows: T[], + columns: ReadonlyArray<SortColumn<T, K>> +) { + const [sort, setSort] = useState<SortState<K> | null>(null); + + const columnsByKey = useMemo(() => { + const map = new Map<K, SortColumn<T, K>>(); + for (const column of columns) { + map.set(column.key, column); + } + return map; + }, [columns]); + + const sortedRows = useMemo(() => { + if (!sort) return rows; + const column = columnsByKey.get(sort.key); + if (!column) return rows; + return sortRows(rows, column, sort.direction); + }, [rows, sort, columnsByKey]); + + const getSortProps = useCallback( + (key: K): TableSortHeaderProps => ({ + sortDirection: sort?.key === key ? sort.direction : null, + onSort: () => + setSort((current) => { + if (!current || current.key !== key) return { key, direction: "asc" }; + if (current.direction === "asc") return { key, direction: "desc" }; + return null; + }), + }), + [sort] + ); + + return { sortedRows, getSortProps, sort }; +} diff --git a/apps/webapp/app/components/query/QueryEditor.tsx b/apps/webapp/app/components/query/QueryEditor.tsx index 37747a77dae..ee230b26f7a 100644 --- a/apps/webapp/app/components/query/QueryEditor.tsx +++ b/apps/webapp/app/components/query/QueryEditor.tsx @@ -10,6 +10,7 @@ import { useCallback, useEffect, useImperativeHandle, + useMemo, useRef, useState, type ReactNode, @@ -72,7 +73,8 @@ import type { action as titleAction } from "~/routes/resources.orgs.$organizatio import type { QueryScope } from "~/services/queryService.server"; import { downloadFile, rowsToCSV, rowsToJSON } from "~/utils/dataExport"; import { organizationBillingPath } from "~/utils/pathBuilder"; -import { querySchemas } from "~/v3/querySchemas"; +import { listableQuerySchemas } from "~/v3/querySchemas"; +import { useFeatures } from "~/hooks/useFeatures"; /** Convert a Date or ISO string to ISO string format */ function toISOString(value: Date | string): string { @@ -187,6 +189,11 @@ const QueryEditorForm = forwardRef< ref ) { const isLoading = fetcher.state === "submitting" || fetcher.state === "loading"; + const { queueMetricsQueryTables } = useFeatures(); + const schemas = useMemo( + () => listableQuerySchemas({ includeQueueMetrics: queueMetricsQueryTables }), + [queueMetricsQueryTables] + ); const [query, setQuery] = useState(defaultQuery); const [scope, setScope] = useState<QueryScope>(defaultScope); const formRef = useRef<HTMLFormElement>(null); @@ -245,7 +252,7 @@ const QueryEditorForm = forwardRef< <TSQLEditor defaultValue={query} onChange={setQuery} - schema={querySchemas} + schema={schemas} linterEnabled={true} showCopyButton={true} showClearButton={true} diff --git a/apps/webapp/app/components/queues/QueueControls.tsx b/apps/webapp/app/components/queues/QueueControls.tsx new file mode 100644 index 00000000000..a96f94da459 --- /dev/null +++ b/apps/webapp/app/components/queues/QueueControls.tsx @@ -0,0 +1,396 @@ +import { AdjustmentsHorizontalIcon, PauseIcon, PlayIcon } from "@heroicons/react/20/solid"; +import { DialogClose } from "@radix-ui/react-dialog"; +import { Form, useNavigation } from "@remix-run/react"; +import type { QueueItem } from "@trigger.dev/core/v3/schemas"; +import { useEffect, useState } from "react"; +import { Button, type ButtonVariant } from "~/components/primitives/Buttons"; +import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog"; +import { FormButtons } from "~/components/primitives/FormButtons"; +import { Hint } from "~/components/primitives/Hint"; +import { Input } from "~/components/primitives/Input"; +import { InputGroup } from "~/components/primitives/InputGroup"; +import { Label } from "~/components/primitives/Label"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { PopoverMenuItem } from "~/components/primitives/Popover"; +import SegmentedControl from "~/components/primitives/SegmentedControl"; +import { Spinner } from "~/components/primitives/Spinner"; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from "~/components/primitives/Tooltip"; + +// Per-queue action controls. Extracted from the Queues list route so the queue detail page can +// reuse them. Both submit a `<Form method="post">` to the current route, so whichever route renders +// them must handle the `queue-pause` / `queue-resume` / `queue-override` / `queue-remove-override` +// actions (see `handleQueueMutationAction` in `~/models/queueMutation.server`). + +export function QueuePauseResumeButton({ + queue, + variant = "tertiary/small", + fullWidth = false, + showTooltip = true, + iconOnly = false, + withQueueName = false, +}: { + /** The "id" here is a friendlyId */ + queue: { id: string; name: string; paused: boolean }; + variant?: ButtonVariant; + fullWidth?: boolean; + showTooltip?: boolean; + /** Icon-only trigger (label moves to the tooltip). For compact placements like the detail-page + * live blocks. */ + iconOnly?: boolean; + /** Render the full "Pause/Resume {name} queue" label instead of the short "Pause"/"Resume". */ + withQueueName?: boolean; +}) { + const [isOpen, setIsOpen] = useState(false); + + const label = queue.paused + ? `Resumes the "${queue.name}" queue so its runs can be dequeued again.` + : `Pauses all runs from being dequeued in the "${queue.name}" queue. Any executing runs will continue to run.`; + + const trigger = showTooltip ? ( + <div> + <TooltipProvider disableHoverableContent={true}> + <Tooltip> + <TooltipTrigger asChild> + <div className={iconOnly ? "cursor-pointer [&_button]:cursor-pointer" : undefined}> + <DialogTrigger asChild> + <Button + type="button" + variant={variant} + LeadingIcon={queue.paused ? PlayIcon : PauseIcon} + leadingIconClassName={queue.paused ? "text-success" : "text-warning"} + fullWidth={fullWidth} + textAlignLeft={fullWidth} + aria-label={label} + className={ + withQueueName + ? queue.paused + ? "border-success/60 text-success [&_span]:text-success hover:border-success" + : "border-warning/60 text-warning [&_span]:text-warning hover:border-warning" + : undefined + } + > + {iconOnly + ? undefined + : withQueueName + ? queue.paused + ? "Resume this queue…" + : "Pause this queue…" + : queue.paused + ? "Resume" + : "Pause"} + </Button> + </DialogTrigger> + </div> + </TooltipTrigger> + <TooltipContent side="right" className={"text-xs"}> + {label} + </TooltipContent> + </Tooltip> + </TooltipProvider> + </div> + ) : ( + <DialogTrigger asChild> + <PopoverMenuItem + icon={queue.paused ? PlayIcon : PauseIcon} + leadingIconClassName={queue.paused ? "text-success" : "text-warning"} + title={queue.paused ? "Resume..." : "Pause..."} + /> + </DialogTrigger> + ); + + return ( + <Dialog open={isOpen} onOpenChange={setIsOpen}> + {trigger} + <DialogContent> + <DialogHeader>{queue.paused ? "Resume queue?" : "Pause queue?"}</DialogHeader> + <div className="flex flex-col gap-3 pt-3"> + <Paragraph> + {queue.paused + ? `This will allow runs to be dequeued in the "${queue.name}" queue again.` + : `This will pause all runs from being dequeued in the "${queue.name}" queue. Any executing runs will continue to run.`} + </Paragraph> + <Form method="post" onSubmit={() => setIsOpen(false)}> + <input + type="hidden" + name="action" + value={queue.paused ? "queue-resume" : "queue-pause"} + /> + <input type="hidden" name="friendlyId" value={queue.id} /> + <FormButtons + confirmButton={ + <Button + type="submit" + shortcut={{ modifiers: ["mod"], key: "enter" }} + variant={queue.paused ? "primary/medium" : "danger/medium"} + LeadingIcon={queue.paused ? PlayIcon : PauseIcon} + > + {queue.paused ? "Resume queue" : "Pause queue"} + </Button> + } + cancelButton={ + <DialogClose asChild> + <Button type="button" variant="secondary/medium"> + Cancel + </Button> + </DialogClose> + } + /> + </Form> + </div> + </DialogContent> + </Dialog> + ); +} + +export function QueueOverrideConcurrencyButton({ + queue, + environmentConcurrencyLimit, + trigger, +}: { + queue: QueueItem & { concurrencyLimitOverridePercent: number | null }; + environmentConcurrencyLimit: number; + /** How to render the dialog trigger. "menu-item" (default) is a PopoverMenuItem for row menus; + * "button" is a standalone labeled button; "icon" is an icon-only button with the label in a + * hover tooltip, for compact placements like the detail-page live blocks. */ + trigger?: "menu-item" | "button" | "icon"; +}) { + const navigation = useNavigation(); + const [isOpen, setIsOpen] = useState(false); + const [mode, setMode] = useState<"absolute" | "percent">( + queue.concurrencyLimitOverridePercent !== null ? "percent" : "absolute" + ); + const [concurrencyLimit, setConcurrencyLimit] = useState<string>( + queue.concurrencyLimit?.toString() ?? environmentConcurrencyLimit.toString() + ); + const [percent, setPercent] = useState<string>( + queue.concurrencyLimitOverridePercent?.toString() ?? "100" + ); + + const isOverridden = !!queue.concurrency?.overriddenAt; + const currentLimit = queue.concurrencyLimit ?? environmentConcurrencyLimit; + + useEffect(() => { + if (navigation.state === "loading" || navigation.state === "idle") { + setIsOpen(false); + } + }, [navigation.state]); + + const isLoading = Boolean( + navigation.formData?.get("action") === "queue-override" || + navigation.formData?.get("action") === "queue-remove-override" + ); + + // Client-side mirror of the backend cap + materialization, so the user sees the resolved value + // and can't submit an above-limit override. + const percentNumber = Number(percent); + const percentValid = Number.isFinite(percentNumber) && percentNumber > 0 && percentNumber <= 100; + const materializedFromPercent = percentValid + ? Math.min( + Math.max(Math.floor((environmentConcurrencyLimit * percentNumber) / 100), 1), + environmentConcurrencyLimit + ) + : null; + + const limitNumber = Number(concurrencyLimit); + const limitOverCap = Number.isFinite(limitNumber) && limitNumber > environmentConcurrencyLimit; + + const submitDisabled = + isLoading || (mode === "percent" ? !percentValid : !concurrencyLimit || limitOverCap); + + const iconLabel = isOverridden ? "Edit override" : "Override limit"; + + return ( + <Dialog open={isOpen} onOpenChange={setIsOpen}> + {trigger === "icon" ? ( + <TooltipProvider disableHoverableContent={true}> + <Tooltip> + <TooltipTrigger asChild> + <div className="cursor-pointer [&_button]:cursor-pointer"> + <DialogTrigger asChild> + <Button + type="button" + variant="secondary/small-icon" + LeadingIcon={AdjustmentsHorizontalIcon} + leadingIconClassName="text-text-dimmed" + aria-label={iconLabel} + /> + </DialogTrigger> + </div> + </TooltipTrigger> + <TooltipContent side="right" className="text-xs"> + {iconLabel} + </TooltipContent> + </Tooltip> + </TooltipProvider> + ) : trigger === "button" ? ( + <TooltipProvider disableHoverableContent={true}> + <Tooltip> + <TooltipTrigger asChild> + <div className="cursor-pointer [&_button]:cursor-pointer"> + <DialogTrigger asChild> + <Button + type="button" + variant="secondary/small" + LeadingIcon={AdjustmentsHorizontalIcon} + leadingIconClassName="text-text-bright" + aria-label={ + isOverridden ? "Edit concurrency override" : "Override concurrency limit" + } + > + {isOverridden ? "Edit override" : "Override limit"} + </Button> + </DialogTrigger> + </div> + </TooltipTrigger> + <TooltipContent side="bottom" className="max-w-[230px] text-xs"> + Give this queue its own concurrency limit instead of the environment default. Set it + as a number or a percentage of the environment limit. + </TooltipContent> + </Tooltip> + </TooltipProvider> + ) : ( + <DialogTrigger asChild> + <PopoverMenuItem + icon={AdjustmentsHorizontalIcon} + title={isOverridden ? "Edit override…" : "Override limit…"} + /> + </DialogTrigger> + )} + <DialogContent> + <DialogHeader> + {isOverridden ? "Edit concurrency override" : "Override concurrency limit"} + </DialogHeader> + <div className="flex flex-col gap-3 pt-3"> + {isOverridden ? ( + <Paragraph variant="small"> + This queue's concurrency limit is currently overridden to {currentLimit}. + {typeof queue.concurrency?.base === "number" && + ` The original limit set in code was ${queue.concurrency.base}.`}{" "} + You can update the override or remove it to restore the{" "} + {typeof queue.concurrency?.base === "number" + ? "limit set in code" + : "environment concurrency limit"} + . + </Paragraph> + ) : ( + <Paragraph variant="small"> + Override this queue's concurrency limit. The current limit is {currentLimit}, which is + set {queue.concurrencyLimit !== null ? "in code" : "by the environment"}. + </Paragraph> + )} + <Form method="post" onSubmit={() => setIsOpen(false)} className="space-y-3"> + <input type="hidden" name="friendlyId" value={queue.id} /> + <input type="hidden" name="mode" value={mode} /> + <InputGroup fullWidth> + <Label htmlFor={mode === "percent" ? "percent" : "concurrencyLimit"}> + Concurrency limit + </Label> + <div className="flex items-center gap-2"> + <div className="flex-1"> + {mode === "percent" ? ( + <Input + type="number" + name="percent" + id="percent" + min="1" + max="100" + step="0.01" + value={percent} + onChange={(e) => setPercent(e.target.value)} + placeholder="100" + autoFocus + accessory={<span className="pr-1 text-text-dimmed">%</span>} + /> + ) : ( + <Input + type="number" + name="concurrencyLimit" + id="concurrencyLimit" + min="0" + max={environmentConcurrencyLimit} + value={concurrencyLimit} + onChange={(e) => setConcurrencyLimit(e.target.value)} + placeholder={currentLimit.toString()} + autoFocus + /> + )} + </div> + <SegmentedControl + name="unit" + value={mode} + className="h-8" + options={[ + { label: "Number", value: "absolute" }, + { label: "Percent", value: "percent" }, + ]} + onChange={(value) => setMode(value === "percent" ? "percent" : "absolute")} + /> + </div> + {mode === "percent" ? ( + <Hint className="tabular-nums"> + {materializedFromPercent !== null + ? `${percentNumber}% = ${materializedFromPercent} concurrent ${ + materializedFromPercent === 1 ? "run" : "runs" + } of the environment's ${environmentConcurrencyLimit}. Recalculates automatically when the environment limit changes.` + : "Enter a percentage between 1 and 100."} + </Hint> + ) : ( + <Hint className={limitOverCap ? "text-warning tabular-nums" : "tabular-nums"}> + {limitOverCap + ? `Can't exceed the environment limit of ${environmentConcurrencyLimit}.` + : `The most concurrent runs this queue can use at once. It can't exceed the environment limit of ${environmentConcurrencyLimit}.`} + </Hint> + )} + </InputGroup> + + <FormButtons + defaultAction={{ + name: "action", + value: "queue-override", + disabled: submitDisabled, + }} + confirmButton={ + <Button + type="submit" + name="action" + value="queue-override" + disabled={submitDisabled} + variant="primary/medium" + LeadingIcon={isLoading && <Spinner color="white" />} + shortcut={{ modifiers: ["mod"], key: "enter" }} + > + {isOverridden ? "Update override" : "Override limit"} + </Button> + } + cancelButton={ + <div className="flex items-center justify-between gap-2"> + {isOverridden && ( + <Button + type="submit" + name="action" + value="queue-remove-override" + disabled={isLoading} + variant="danger/medium" + > + Remove override + </Button> + )} + <DialogClose asChild> + <Button type="button" variant="secondary/medium"> + Cancel + </Button> + </DialogClose> + </div> + } + /> + </Form> + </div> + </DialogContent> + </Dialog> + ); +} diff --git a/apps/webapp/app/components/queues/QueueMetricCards.tsx b/apps/webapp/app/components/queues/QueueMetricCards.tsx new file mode 100644 index 00000000000..4349ea80c36 --- /dev/null +++ b/apps/webapp/app/components/queues/QueueMetricCards.tsx @@ -0,0 +1,462 @@ +import { useEffect, useMemo, useState, type ReactNode } from "react"; +import { buildActivityTimeAxis } from "~/components/primitives/charts/activityTimeAxis"; +import { + Chart, + type ChartConfig, + type ChartState, +} from "~/components/primitives/charts/ChartCompound"; +import { ChartCard } from "~/components/primitives/charts/ChartCard"; +import { MiniLineChart } from "~/components/metrics/MiniLineChart"; +import { + useMetricResourceQuery, + type MetricResourceTimeRange, +} from "~/hooks/useMetricResourceQuery"; +import { Header3 } from "~/components/primitives/Headers"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { InfoIconTooltip } from "~/components/primitives/Tooltip"; +import { useSearchParams } from "~/hooks/useSearchParam"; +import { cn } from "~/utils/cn"; +import { formatNumberCompact } from "~/utils/numberFormatter"; + +// Shared building blocks for queue-metric UI (queue detail page, task detail page, +// run inspector). All CH-derived data is fetched client-side through useQueueMetric +// so pages render instantly; loaders only supply live counts and identifiers. + +export const QUEUE_METRIC_COLORS = { + running: "var(--color-queues)", + limit: "#4D525B", + queued: "var(--color-queues)", + p50: "#22D3EE", + p95: "#F59E0B", + p99: "#EF4444", + throttled: "#F59E0B", + ckKeys: "#34D399", + ckWait: "#F59E0B", +}; + +export const QUEUE_METRICS_DEFAULT_PERIOD = "1d"; + +export type QueueMetricIds = { + organizationId: string; + projectId: string; + environmentId: string; +}; + +export type QueueMetricTimeRange = MetricResourceTimeRange; + +export function useQueueMetric( + query: string, + opts: { + ids: QueueMetricIds; + timeRange: QueueMetricTimeRange; + queueName: string; + fillGaps?: boolean; + /** Match the host page's TimeFilter default (e.g. "7d" on task detail). */ + defaultPeriod?: string; + /** Poll ClickHouse on this cadence (ms). Omit to use the query's default interval. */ + refreshIntervalMs?: number; + } +) { + return useMetricResourceQuery(query, { + ...opts.ids, + timeRange: opts.timeRange, + defaultPeriod: opts.defaultPeriod ?? QUEUE_METRICS_DEFAULT_PERIOD, + queues: [opts.queueName], + fillGaps: opts.fillGaps, + refreshIntervalMs: opts.refreshIntervalMs, + }); +} + +export function toNumber(value: number | string | null | undefined): number { + const n = typeof value === "number" ? value : Number(value); + return Number.isFinite(n) ? n : 0; +} + +export function clickhouseTimeToMs(value: unknown): number { + const s = String(value).replace(" ", "T"); + return Date.parse(s.endsWith("Z") ? s : `${s}Z`); +} + +export function formatWaitMs(ms: number): string { + if (ms < 1000) return `${Math.round(ms)}ms`; + if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`; + if (ms < 3_600_000) return `${(ms / 60_000).toFixed(1)}m`; + return `${(ms / 3_600_000).toFixed(1)}h`; +} + +export type QueueMetricSeriesConfig = { key: string; label: string; color: string }; + +type QueueMetricChartProps = { + query: string; + series: QueueMetricSeriesConfig[]; + ids: QueueMetricIds; + timeRange: QueueMetricTimeRange; + queueName: string; + valueFormat?: (value: number) => string; + fillGaps?: boolean; + defaultPeriod?: string; + /** Recolor a series warning where it drops below another (e.g. started below enqueued). */ + warningOverlay?: { series: string; below: string } | { series: string; atOrAbove: string }; + /** + * Series whose leading zeros should be back-filled with the first real value. Gauge series that + * are only emitted while the queue is active (e.g. the concurrency `limit`) read as 0 before the + * first emission — carry-forward has nothing to carry yet — which draws a false 0→N step. These + * are config values that existed all along, so carry the first value backward instead. + */ + carryBackfill?: string[]; + /** Show the series legend below the chart (use for multi-series charts). */ + showLegend?: boolean; + /** + * Recolour a series' stroke above a threshold with a gradient split (colour only above the + * line). `value` sets a constant threshold; `valueFromSeries` reads a (roughly constant) + * threshold off another series — e.g. the concurrency limit. `series` targets which line is + * recoloured; the others keep their own colour. + */ + thresholdStroke?: { + aboveColor: string; + series?: string; + value?: number; + valueFromSeries?: string; + }; + /** Reports whether the chart has data to plot (false once it settles on the "no activity" state), + * so a wrapping card can hide the legend to match. */ + onHasDataChange?: (hasData: boolean) => void; +}; + +// Bare chart (no card chrome) so it can live inside a shared card, e.g. a tabbed panel. +export function QueueMetricChart({ + query, + series, + ids, + timeRange, + queueName, + valueFormat, + fillGaps, + defaultPeriod, + warningOverlay, + carryBackfill, + thresholdStroke, + onHasDataChange, +}: QueueMetricChartProps) { + const { rows, showLoading, failed } = useQueueMetric(query, { + ids, + timeRange, + queueName, + fillGaps, + defaultPeriod, + }); + + const data = useMemo(() => { + const points = rows + .map((r) => { + const point: { bucket: number } & Record<string, number> = { + bucket: clickhouseTimeToMs(r.t), + }; + for (const s of series) point[s.key] = toNumber(r[s.key]); + return point; + }) + .filter((p) => Number.isFinite(p.bucket)); + + // Back-fill leading zeros for config gauges (see `carryBackfill`): find the first positive + // value and carry it back over the earlier buckets so the line doesn't start at a false 0. + if (carryBackfill?.length) { + for (const key of carryBackfill) { + const first = points.findIndex((p) => p[key] > 0); + if (first > 0) { + const value = points[first]![key]!; + for (let i = 0; i < first; i++) points[i]![key] = value; + } + } + } + return points; + }, [rows, series, carryBackfill]); + + const chartConfig = useMemo(() => { + const cfg: ChartConfig = {}; + for (const s of series) cfg[s.key] = { label: s.label, color: s.color }; + return cfg; + }, [series]); + + const { tickFormatter, tooltipLabelFormatter } = useMemo( + () => buildActivityTimeAxis(data), + [data] + ); + + // Resolve the threshold value: a constant, or the max of another series (e.g. the limit line, + // which is effectively constant). A gradient split then colours the target series only above it. + // `valueFromSeries` targets integer-count series (concurrency limit), so split half a unit below + // the limit — that way the line renders warning *at or above* the limit (saturated), matching + // "turns yellow at the limit", rather than only when it strictly exceeds it. + const resolvedThresholdStroke = useMemo(() => { + if (!thresholdStroke) return undefined; + let value = thresholdStroke.value; + if (value == null && thresholdStroke.valueFromSeries) { + let max = -Infinity; + for (const p of data) { + const v = Number(p[thresholdStroke.valueFromSeries]); + if (Number.isFinite(v) && v > max) max = v; + } + value = max > 0 ? max - 0.5 : undefined; + } + if (value == null || !Number.isFinite(value)) return undefined; + return { value, aboveColor: thresholdStroke.aboveColor, series: thresholdStroke.series }; + }, [thresholdStroke, data]); + + const state: ChartState = showLoading ? "loading" : failed ? "invalid" : undefined; + + // Report data presence so a wrapping card can hide its legend when the chart settles on the + // "no activity" state. Only report once loaded, so the legend stays put while loading. + useEffect(() => { + if (!showLoading) onHasDataChange?.(!failed && data.length > 0); + }, [showLoading, failed, data.length, onHasDataChange]); + + return ( + <Chart.Root + config={chartConfig} + data={data} + dataKey="bucket" + series={series.map((s) => s.key)} + state={state} + fillContainer + > + <Chart.Line + lineType="monotone" + xAxisProps={{ tickFormatter }} + yAxisProps={valueFormat ? { tickFormatter: (v: number) => valueFormat(v) } : undefined} + tooltipLabelFormatter={tooltipLabelFormatter} + tooltipValueFormatter={valueFormat} + warningOverlay={warningOverlay} + thresholdStroke={resolvedThresholdStroke} + /> + </Chart.Root> + ); +} + +export function QueueMetricChartCard({ + title, + info, + titleAccessory, + className, + extraLegend, + ...chart +}: QueueMetricChartProps & { + title: string; + info?: ReactNode; + /** Extra content rendered after the info icon inside the title row (e.g. a live readout). */ + titleAccessory?: ReactNode; + className?: string; + /** Extra legend entries appended after the series — e.g. a warning state that isn't its own + * series (the orange "over threshold" colour). */ + extraLegend?: Array<{ color: string; label: string }>; +}) { + // Hide the legend once the chart settles on the "no activity" state (reported by the chart). + const [hasData, setHasData] = useState(true); + return ( + <div className={className ?? "h-64"}> + <ChartCard + title={ + <span className="flex flex-col gap-1"> + <span className="flex items-center gap-1"> + {title} + {info ? ( + <InfoIconTooltip + content={info} + contentClassName="max-w-[230px]" + disableHoverableContent + /> + ) : null} + {titleAccessory} + </span> + {/* Inline legend below the title (swatch + label per series), matching the list-page + charts — instead of the Chart.Root legend with per-series totals. */} + {chart.showLegend && + hasData && + (chart.series.length > 0 || (extraLegend?.length ?? 0) > 0) ? ( + <span className="flex flex-wrap items-center gap-2"> + {chart.series.map((s) => ( + <span + key={s.key} + className="flex items-center gap-1 text-xs font-normal text-text-dimmed" + > + <span className="size-2.5 rounded-[2px]" style={{ backgroundColor: s.color }} /> + {s.label} + </span> + ))} + {extraLegend?.map((e) => ( + <span + key={e.label} + className="flex items-center gap-1 text-xs font-normal text-text-dimmed" + > + <span className="size-2.5 rounded-[2px]" style={{ backgroundColor: e.color }} /> + {e.label} + </span> + ))} + </span> + ) : null} + </span> + } + > + <QueueMetricChart {...chart} onHasDataChange={setHasData} /> + </ChartCard> + </div> + ); +} + +export type QueueLiveCounts = { queued: number; running: number }; + +// Compact two-line stat block for task detail sidebars: live counts from the loader, +// then delay p95 + peak backlog over the page's TimeFilter range. +export function QueueSidebarStats({ + live, + ids, + queueName, + defaultPeriod, +}: { + live: QueueLiveCounts; + ids: QueueMetricIds; + queueName: string; + defaultPeriod?: string; +}) { + const { value } = useSearchParams(); + const timeRange: QueueMetricTimeRange = { + period: value("period") ?? null, + from: value("from") ?? null, + to: value("to") ?? null, + }; + + const { rows, showLoading } = useQueueMetric( + `SELECT max(max_queued) AS peak_queued,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS worst_p95\nFROM queue_metrics`, + { ids, timeRange, queueName, defaultPeriod } + ); + const row = rows[0]; + const worstP95 = row ? toNumber(row.worst_p95) : 0; + const peakQueued = row ? toNumber(row.peak_queued) : 0; + + return ( + <> + <Paragraph variant="extra-small" className="tabular-nums text-text-dimmed"> + Queued now {live.queued.toLocaleString()} · Running now {live.running.toLocaleString()} + </Paragraph> + <Paragraph variant="extra-small" className="tabular-nums text-text-dimmed"> + {showLoading || !row + ? "…" + : `Delay p95 ${worstP95 > 0 ? formatWaitMs(worstP95) : "–"} · Peak backlog ${peakQueued.toLocaleString()}`} + </Paragraph> + </> + ); +} + +// A compact stat card with a recent trend sparkline underneath, for the run inspector. +// The headline is a live "now" value from the loader; the sparkline pulls its own series. +const SPARKLINE_PERIOD = "30m"; + +export function QueueSparklineStat({ + title, + info, + query, + color, + ids, + queueName, + formatPeak, + unitLabel, + chartHeight, +}: { + title: string; + /** Tooltip text under the info icon next to the title (matches the queue page copy). */ + info?: ReactNode; + query: string; + color: string; + ids: QueueMetricIds; + queueName: string; + formatPeak?: (peak: number) => string; + /** Unit shown in the per-bucket hover tooltip (e.g. queued, ms). */ + unitLabel?: { singular: string; plural: string }; + /** Plot height in px. Defaults to the shared mini-chart height. */ + chartHeight?: number; +}) { + const timeRange: QueueMetricTimeRange = { period: SPARKLINE_PERIOD, from: null, to: null }; + const { rows } = useQueueMetric(query, { + ids, + timeRange, + queueName, + fillGaps: true, + defaultPeriod: SPARKLINE_PERIOD, + }); + + const { data, throttled, bucketStartMs, bucketIntervalMs, peak } = useMemo(() => { + const points = rows + .map((r) => ({ + bucket: clickhouseTimeToMs(r.t), + v: toNumber(r.v), + // Present only when the query selects it (Backlog); 0 elsewhere so no overlay draws. + throttled: toNumber(r.throttled), + })) + .filter((p) => Number.isFinite(p.bucket)) + .sort((a, b) => a.bucket - b.bucket); + return { + data: points.map((p) => p.v), + throttled: points.map((p) => p.throttled), + bucketStartMs: points[0]?.bucket, + bucketIntervalMs: points.length > 1 ? points[1]!.bucket - points[0]!.bucket : undefined, + peak: points.reduce((m, p) => Math.max(m, p.v), 0), + }; + }, [rows]); + + return ( + <div className="flex flex-col gap-2"> + <div className="flex items-center gap-1"> + <Header3 className="leading-6">{title}</Header3> + {info || (data.length > 0 && peak > 0) ? ( + <InfoIconTooltip + content={ + <div className="flex flex-col gap-1"> + {info ? <span>{info}</span> : null} + {data.length > 0 && peak > 0 ? ( + <span className="tabular-nums text-text-dimmed"> + Peak {formatPeak ? formatPeak(peak) : formatNumberCompact(peak)} + </span> + ) : null} + </div> + } + contentClassName="max-w-[230px]" + disableHoverableContent + /> + ) : null} + </div> + <MiniLineChart + data={data} + throttled={throttled} + bucketStartMs={bucketStartMs} + bucketIntervalMs={bucketIntervalMs} + color={color} + unitLabel={unitLabel} + height={chartHeight} + fillWidth + showPeak={false} + /> + </div> + ); +} + +export function QueueMetricStat({ + label, + value, + className, + loading, +}: { + label: string; + value: string; + className?: string; + loading?: boolean; +}) { + return ( + <div className="rounded-sm border border-grid-dimmed bg-background-bright px-3 py-2"> + <div className="text-xs text-text-dimmed">{label}</div> + {loading ? ( + <div className="mt-1 h-6 w-12 animate-pulse rounded bg-grid-bright/50" /> + ) : ( + <div className={cn("text-2xl tabular-nums text-text-bright", className)}>{value}</div> + )} + </div> + ); +} diff --git a/apps/webapp/app/entry.server.tsx b/apps/webapp/app/entry.server.tsx index da1ab5a433a..4511deb9966 100644 --- a/apps/webapp/app/entry.server.tsx +++ b/apps/webapp/app/entry.server.tsx @@ -10,6 +10,7 @@ import { PassThrough } from "stream"; import { initMollifierDrainerWorker } from "~/v3/mollifierDrainerWorker.server"; import { initMollifierStaleSweepWorker } from "~/v3/mollifierStaleSweepWorker.server"; import { initBillingLimitWorker } from "~/v3/billingLimitWorker.server"; +import { initQueueMetricsConsumer, initQueueMetricsEmitter } from "~/v3/queueMetrics.server"; import { bootstrap } from "./bootstrap"; import { LocaleContextProvider } from "./components/primitives/LocaleProvider"; import type { OperatingSystemPlatform } from "./components/primitives/OperatingSystemProvider"; @@ -234,6 +235,8 @@ export const handleError = wrapHandleErrorWithSentry((error, { request }) => { initMollifierDrainerWorker(); initMollifierStaleSweepWorker(); initBillingLimitWorker(); +initQueueMetricsEmitter(); +initQueueMetricsConsumer(); bootstrap().catch((error) => { logError(error); diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index f2240bd6813..24ed0833d8e 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -917,6 +917,37 @@ const EnvironmentSchema = z RUN_ENGINE_REUSE_SNAPSHOT_COUNT: z.coerce.number().int().default(0), RUN_ENGINE_MAXIMUM_ENV_COUNT: z.coerce.number().int().optional(), RUN_ENGINE_RUN_QUEUE_SHARD_COUNT: z.coerce.number().int().default(4), + // Queue metrics ingestion (Redis Stream -> ClickHouse). The runtime on/off is the + // `queue_metrics:enabled` Redis key; these gate emitter construction + consumer boot. + QUEUE_METRICS_EMIT_ENABLED: z.string().default("0"), + QUEUE_METRICS_CONSUMER_ENABLED: z.string().default("0"), + QUEUE_METRICS_STREAM_SHARD_COUNT: z.coerce.number().int().default(4), + QUEUE_METRICS_CONSUMER_BATCH_SIZE: z.coerce.number().int().default(1000), + // Counter stream (exact counts, loss-intolerant). Unset host => the run-queue Redis; + // set it to a dedicated instance so counter backlog never competes with the run queue. + QUEUE_METRICS_REDIS_HOST: z.string().optional(), + QUEUE_METRICS_REDIS_PORT: z.coerce.number().optional(), + QUEUE_METRICS_REDIS_USERNAME: z.string().optional(), + QUEUE_METRICS_REDIS_PASSWORD: z.string().optional(), + QUEUE_METRICS_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"), + // Default depends on where the stream lives: see metricsDefinition() in + // queueMetrics.server.ts (2M on the shared run-queue Redis, 8M on a dedicated one). + QUEUE_METRICS_COUNTER_STREAM_MAXLEN: z.coerce.number().int().optional(), + // TTL (seconds) on the per-(queue,op) cumulative odometer key, refreshed on every write. + // Idle-past-TTL queues purge and self-heal (restart from 1) on return; default 7 days. + QUEUE_METRICS_COUNTER_ODOMETER_TTL_SECONDS: z.coerce.number().int().default(604_800), + // Per-env distinct queue_name cap (0 = unlimited); overflow maps to "__overflow__". + QUEUE_METRICS_MAX_QUEUE_NAMES_PER_ENV: z.coerce.number().int().default(1000), + QUEUE_METRICS_MAX_CONCURRENCY_KEYS_PER_QUEUE: z.coerce.number().int().default(10_000), + // Fraction (0..1) of ops that emit a gauge; counters are never sampled. Dial below 1 + // only if EngineCPU is too high in slow-path-heavy regions (hurts low-traffic queues). + QUEUE_METRICS_GAUGE_SAMPLE_RATE: z.coerce.number().min(0).max(1).default(1), + /** + * Lists the queue-metrics tables in the Query page, its schema docs, the schema API and the + * AI query context. Off by default so the tables are not advertised before ingestion is + * enabled. Listing only: a query naming one of these tables still runs either way. + */ + QUEUE_METRICS_QUERY_TABLES_VISIBLE: z.string().default("0"), RUN_ENGINE_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000), RUN_ENGINE_RETRY_WARM_START_THRESHOLD_MS: z.coerce.number().int().default(30_000), RUN_ENGINE_PROCESS_WORKER_QUEUE_DEBOUNCE_MS: z.coerce.number().int().default(200), @@ -1927,6 +1958,22 @@ const EnvironmentSchema = z .enum(["log", "error", "warn", "info", "debug"]) .default("info"), RUNS_LIST_CLICKHOUSE_COMPRESSION_REQUEST: z.string().default("1"), + /** + * Dedicated ClickHouse service for queue metrics: the ingestion consumer's inserts and every + * queue-metrics read (dashboards, queue pages, run inspector, health report) go through it, so + * metrics traffic never competes with runs-list or trace reads. Unset keeps the previous + * wiring: inserts on CLICKHOUSE_URL, reads on the query pool. + */ + QUEUE_METRICS_CLICKHOUSE_URL: z.string().optional(), + /** Reader split so the consumer's inserts can never land on a read endpoint. Defaults to QUEUE_METRICS_CLICKHOUSE_URL. */ + QUEUE_METRICS_CLICKHOUSE_READER_URL: z.string().optional(), + QUEUE_METRICS_CLICKHOUSE_KEEP_ALIVE_ENABLED: z.string().default("1"), + QUEUE_METRICS_CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS: z.coerce.number().int().optional(), + QUEUE_METRICS_CLICKHOUSE_MAX_OPEN_CONNECTIONS: z.coerce.number().int().default(10), + QUEUE_METRICS_CLICKHOUSE_LOG_LEVEL: z + .enum(["log", "error", "warn", "info", "debug"]) + .default("info"), + QUEUE_METRICS_CLICKHOUSE_COMPRESSION_REQUEST: z.string().default("1"), EVENTS_CLICKHOUSE_BATCH_SIZE: z.coerce.number().int().default(1000), EVENTS_CLICKHOUSE_FLUSH_INTERVAL_MS: z.coerce.number().int().default(1000), METRICS_CLICKHOUSE_BATCH_SIZE: z.coerce.number().int().default(10000), diff --git a/apps/webapp/app/features.server.ts b/apps/webapp/app/features.server.ts index 36a134dfe99..bf092b314c8 100644 --- a/apps/webapp/app/features.server.ts +++ b/apps/webapp/app/features.server.ts @@ -4,6 +4,7 @@ import { requestUrl } from "./utils/requestUrl.server"; export type TriggerFeatures = { isManagedCloud: boolean; hasPrivateConnections: boolean; + queueMetricsQueryTables: boolean; }; function isManagedCloud(host: string): boolean { @@ -26,6 +27,7 @@ function featuresForHost(host: string): TriggerFeatures { return { isManagedCloud: isManagedCloud(host), hasPrivateConnections: hasPrivateConnections(host), + queueMetricsQueryTables: env.QUEUE_METRICS_QUERY_TABLES_VISIBLE === "1", }; } diff --git a/apps/webapp/app/hooks/useFeatures.ts b/apps/webapp/app/hooks/useFeatures.ts index 5ae3dfbf34c..b6835e18eca 100644 --- a/apps/webapp/app/hooks/useFeatures.ts +++ b/apps/webapp/app/hooks/useFeatures.ts @@ -5,5 +5,11 @@ import type { TriggerFeatures } from "~/features.server"; export function useFeatures(): TriggerFeatures { const routeMatch = useTypedRouteLoaderData<typeof loader>("root"); - return routeMatch?.features ?? { isManagedCloud: false, hasPrivateConnections: false }; + return ( + routeMatch?.features ?? { + isManagedCloud: false, + hasPrivateConnections: false, + queueMetricsQueryTables: false, + } + ); } diff --git a/apps/webapp/app/hooks/useMetricResourceQuery.ts b/apps/webapp/app/hooks/useMetricResourceQuery.ts new file mode 100644 index 00000000000..a4f26bfd9d2 --- /dev/null +++ b/apps/webapp/app/hooks/useMetricResourceQuery.ts @@ -0,0 +1,165 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useInterval } from "./useInterval"; + +export type MetricResourceRow = Record<string, number | string | null>; + +type MetricResourceResponse = + | { success: true; data: { rows: MetricResourceRow[] } } + | { success: false; error: string }; + +export type MetricResourceTimeRange = { + period: string | null; + from: string | null; + to: string | null; +}; + +export type MetricResourceQueryOptions = { + organizationId: string; + projectId: string; + environmentId: string; + timeRange: MetricResourceTimeRange; + defaultPeriod: string; + queues?: string[]; + fillGaps?: boolean; + refreshIntervalMs?: number; +}; + +// Module-level cache of the last successful rows per query signature. Lets a remounted chart +// (switching tabs, or navigating back to the queues list) paint its previous data immediately +// instead of flashing a loading skeleton every time, while it revalidates in the background. +// Bounded so it can't grow without limit over a long session. +const responseCache = new Map<string, MetricResourceRow[]>(); +const RESPONSE_CACHE_MAX = 200; + +function cacheSet(key: string, rows: MetricResourceRow[]) { + // Re-insert so the key becomes the most-recently-used (Map preserves insertion order). + responseCache.delete(key); + responseCache.set(key, rows); + if (responseCache.size > RESPONSE_CACHE_MAX) { + const oldest = responseCache.keys().next().value; + if (oldest !== undefined) responseCache.delete(oldest); + } +} + +/** + * Client-fetch a TRQL query from the metric resource route (like the dashboard + * widgets): own loading state, interval + on-focus refresh, abort on change/unmount. + * + * Successful results are cached per query signature, so a chart that remounts (tab switch, or + * back-navigation to the queues list) shows its last data immediately and revalidates in the + * background rather than flashing a loading skeleton. + */ +export function useMetricResourceQuery(query: string, opts: MetricResourceQueryOptions) { + const { + organizationId, + projectId, + environmentId, + defaultPeriod, + fillGaps, + refreshIntervalMs = 60_000, + } = opts; + const { period, from, to } = opts.timeRange; + const queuesKey = opts.queues && opts.queues.length > 0 ? opts.queues.join(",") : undefined; + const resolvedPeriod = period ?? (from || to ? null : defaultPeriod); + const cacheKey = useMemo( + () => + [ + organizationId, + projectId, + environmentId, + resolvedPeriod ?? "", + from ?? "", + to ?? "", + fillGaps ? 1 : 0, + queuesKey ?? "", + query, + ].join("|"), + [organizationId, projectId, environmentId, resolvedPeriod, from, to, fillGaps, queuesKey, query] + ); + + const [rows, setRows] = useState<MetricResourceRow[] | null>( + () => responseCache.get(cacheKey) ?? null + ); + const [isLoading, setIsLoading] = useState(true); + const [failed, setFailed] = useState(false); + const abortRef = useRef<AbortController | null>(null); + const loadedKeyRef = useRef<string | null>(null); + + const load = useCallback(() => { + abortRef.current?.abort(); + const controller = new AbortController(); + abortRef.current = controller; + // On a new query signature the rows and failure on screen belong to a different query. Paint + // this key's cached rows if we have them (no skeleton on remount / back-navigation), otherwise + // clear them so a genuinely new query shows a loading state instead of another query's data. + // Interval and on-focus refreshes reuse the same signature, so they keep what's on screen. + if (loadedKeyRef.current !== cacheKey) { + loadedKeyRef.current = cacheKey; + setRows(responseCache.get(cacheKey) ?? null); + setFailed(false); + } + setIsLoading(true); + fetch("/resources/metric", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + query, + scope: "environment", + period: resolvedPeriod, + from, + to, + fillGaps: !!fillGaps, + organizationId, + projectId, + environmentId, + ...(queuesKey !== undefined ? { queues: queuesKey.split(",") } : {}), + }), + signal: controller.signal, + }) + .then((res) => res.json() as Promise<MetricResourceResponse>) + .then((data) => { + if (controller.signal.aborted) return; + if (data.success) { + cacheSet(cacheKey, data.data.rows); + setRows(data.data.rows); + setFailed(false); + } else { + setFailed(true); + } + setIsLoading(false); + }) + .catch((error) => { + if (error instanceof DOMException && error.name === "AbortError") return; + if (!controller.signal.aborted) { + setFailed(true); + setIsLoading(false); + } + }); + }, [ + cacheKey, + query, + resolvedPeriod, + from, + to, + fillGaps, + organizationId, + projectId, + environmentId, + queuesKey, + ]); + + useEffect(() => { + load(); + return () => abortRef.current?.abort(); + }, [load]); + + useInterval({ + interval: refreshIntervalMs, + onLoad: false, + onFocus: true, + pauseWhenHidden: true, + callback: load, + }); + + return { rows: rows ?? [], isLoading, showLoading: isLoading && !rows, failed }; +} diff --git a/apps/webapp/app/models/queueMutation.server.ts b/apps/webapp/app/models/queueMutation.server.ts new file mode 100644 index 00000000000..5e4dbcd90e4 --- /dev/null +++ b/apps/webapp/app/models/queueMutation.server.ts @@ -0,0 +1,158 @@ +import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; +import { getUserById } from "~/models/user.server"; +import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server"; +import { + isValidQueueOverridePercent, + MAX_QUEUE_OVERRIDE_PERCENT, + MIN_QUEUE_OVERRIDE_PERCENT, +} from "~/v3/services/concurrencySystem.server"; +import { PauseQueueService } from "~/v3/services/pauseQueue.server"; + +/** + * Handles the per-queue mutating form actions (pause/resume/override/remove-override) shared by the + * Queues list route and the queue detail route. Returns a redirect Response for one of those four + * actions, or `null` if `formData`'s `action` isn't one of them (so the caller can fall through to + * its own action handling). `redirectPath` is where to send the user afterwards — the caller passes + * its own page so a mutation from the detail page stays on the detail page. + */ +export async function handleQueueMutationAction({ + request, + environment, + userId, + formData, + redirectPath, +}: { + request: Request; + environment: AuthenticatedEnvironment; + userId: string; + formData: FormData; + redirectPath: string; +}): Promise<Response | null> { + const action = formData.get("action"); + + switch (action) { + case "queue-pause": + case "queue-resume": { + const friendlyId = formData.get("friendlyId"); + if (!friendlyId) { + return redirectWithErrorMessage(redirectPath, request, "Queue ID is required"); + } + + const queueService = new PauseQueueService(); + const result = await queueService.call( + environment, + friendlyId.toString(), + action === "queue-pause" ? "paused" : "resumed" + ); + + if (!result.success) { + return redirectWithErrorMessage( + redirectPath, + request, + result.error ?? `Failed to ${action === "queue-pause" ? "pause" : "resume"} queue` + ); + } + + return redirectWithSuccessMessage( + redirectPath, + request, + `Queue ${action === "queue-pause" ? "paused" : "resumed"}` + ); + } + case "queue-override": { + const friendlyId = formData.get("friendlyId"); + const mode = formData.get("mode") === "percent" ? "percent" : "absolute"; + + if (!friendlyId) { + return redirectWithErrorMessage(redirectPath, request, "Queue ID is required"); + } + + // The dialog submits either a `percent` of the environment limit or an absolute `limit`, + // depending on the unit toggle. Build the matching override shape for the service. + let override: number | { limit: number } | { percent: number }; + if (mode === "percent") { + const percentValue = formData.get("percent"); + if (!percentValue) { + return redirectWithErrorMessage(redirectPath, request, "Percentage is required"); + } + const percentNumber = Number(percentValue.toString()); + if (!isValidQueueOverridePercent(percentNumber)) { + return redirectWithErrorMessage( + redirectPath, + request, + `Percentage must be greater than ${MIN_QUEUE_OVERRIDE_PERCENT} and less than or equal to ${MAX_QUEUE_OVERRIDE_PERCENT}` + ); + } + override = { percent: percentNumber }; + } else { + const concurrencyLimit = formData.get("concurrencyLimit"); + if (!concurrencyLimit) { + return redirectWithErrorMessage(redirectPath, request, "Concurrency limit is required"); + } + const limitNumber = parseInt(concurrencyLimit.toString(), 10); + if (isNaN(limitNumber) || limitNumber < 0) { + return redirectWithErrorMessage( + redirectPath, + request, + "Concurrency limit must be a valid number" + ); + } + override = { limit: limitNumber }; + } + + const user = await getUserById(userId); + if (!user) { + return redirectWithErrorMessage(redirectPath, request, "User not found"); + } + + const result = await concurrencySystem.queues.overrideQueueConcurrencyLimit( + environment, + friendlyId.toString(), + override, + user + ); + + if (!result.isOk()) { + // Surface the service's specific message (e.g. the above-cap rejection) instead of a + // generic failure so the user learns why the override was refused. + const error = result.error; + const message = + "message" in error && typeof error.message === "string" + ? error.message + : "Failed to override queue concurrency limit"; + return redirectWithErrorMessage(redirectPath, request, message); + } + + return redirectWithSuccessMessage( + redirectPath, + request, + "Queue concurrency limit overridden" + ); + } + case "queue-remove-override": { + const friendlyId = formData.get("friendlyId"); + + if (!friendlyId) { + return redirectWithErrorMessage(redirectPath, request, "Queue ID is required"); + } + + const result = await concurrencySystem.queues.resetConcurrencyLimit( + environment, + friendlyId.toString() + ); + + if (!result.isOk()) { + return redirectWithErrorMessage( + redirectPath, + request, + "Failed to reset queue concurrency limit" + ); + } + + return redirectWithSuccessMessage(redirectPath, request, "Queue concurrency limit reset"); + } + default: + return null; + } +} diff --git a/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts b/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts index 971fc9a3033..d831568248d 100644 --- a/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts +++ b/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts @@ -550,7 +550,186 @@ const llmDashboard: BuiltInDashboard = { }, }; -const builtInDashboards: BuiltInDashboard[] = [overviewDashboard, llmDashboard]; +const queuesDashboard: BuiltInDashboard = { + key: "queues", + title: "Queues", + filters: ["queues"], + layout: { + version: "1", + layout: [ + { i: "env-used", x: 0, y: 0, w: 3, h: 4 }, + { i: "env-limit", x: 3, y: 0, w: 3, h: 4 }, + { i: "env-avail", x: 6, y: 0, w: 3, h: 4 }, + { i: "env-sat", x: 9, y: 0, w: 3, h: 4 }, + { i: "sat-time", x: 0, y: 4, w: 6, h: 9 }, + { i: "used-limit", x: 6, y: 4, w: 6, h: 9 }, + { i: "t-pressure", x: 0, y: 13, w: 12, h: 2, minH: 2, maxH: 2 }, + { i: "pressure", x: 0, y: 15, w: 12, h: 11 }, + { i: "t-trends", x: 0, y: 26, w: 12, h: 2, minH: 2, maxH: 2 }, + { i: "running-q", x: 0, y: 28, w: 6, h: 9 }, + { i: "queued-q", x: 6, y: 28, w: 6, h: 9 }, + { i: "throttled-q", x: 0, y: 37, w: 6, h: 9 }, + { i: "throughput", x: 6, y: 37, w: 6, h: 9 }, + { i: "wait-pct", x: 0, y: 46, w: 12, h: 9 }, + ], + widgets: { + "env-used": { + title: "Concurrency in use", + query: `SELECT argMax(max_env_running, bucket_start) AS in_use\nFROM env_metrics`, + display: { type: "bignumber", column: "in_use", aggregation: "max", abbreviate: false }, + }, + "env-limit": { + title: "Environment limit", + query: `SELECT argMax(max_env_limit, bucket_start) AS env_limit\nFROM env_metrics`, + display: { type: "bignumber", column: "env_limit", aggregation: "max", abbreviate: false }, + }, + "env-avail": { + title: "Available slots", + query: `SELECT argMax(max_env_limit, bucket_start) - argMax(max_env_running, bucket_start) AS available\nFROM env_metrics`, + display: { type: "bignumber", column: "available", aggregation: "max", abbreviate: false }, + }, + "env-sat": { + title: "Env saturation", + query: `SELECT round(argMax(max_env_running, bucket_start) * 100.0 / nullIf(argMax(max_env_limit, bucket_start), 0), 1) AS saturation\nFROM env_metrics`, + display: { + type: "bignumber", + column: "saturation", + aggregation: "max", + abbreviate: false, + suffix: "%", + }, + }, + "sat-time": { + title: "Environment saturation over time", + query: `SELECT timeBucket() AS t,\n round(max(max_env_running) * 100.0 / nullIf(max(max_env_limit), 0), 1) AS saturation\nFROM env_metrics\nGROUP BY t\nORDER BY t`, + display: { + type: "chart", + chartType: "line", + xAxisColumn: "t", + yAxisColumns: ["saturation"], + groupByColumn: null, + stacked: false, + sortByColumn: null, + sortDirection: "asc", + aggregation: "max", + }, + }, + "used-limit": { + title: "Concurrency used vs limit", + query: `SELECT timeBucket() AS t,\n max(max_env_running) AS used,\n max(max_env_limit) AS limit\nFROM env_metrics\nGROUP BY t\nORDER BY t`, + // Single-series gauge: carry the last known used/limit across idle buckets instead of dropping to 0. + fillGaps: true, + display: { + type: "chart", + chartType: "line", + xAxisColumn: "t", + yAxisColumns: ["used", "limit"], + groupByColumn: null, + stacked: false, + sortByColumn: null, + sortDirection: "asc", + aggregation: "max", + }, + }, + "t-pressure": { title: "Queue pressure", query: "", display: { type: "title" } }, + pressure: { + title: "Queue pressure", + query: `SELECT queue,\n argMax(max_running, bucket_start) AS running,\n argMax(max_queued, bucket_start) AS queued,\n argMax(max_limit, bucket_start) AS limit,\n running + queued AS demand,\n max(max_queued) AS peak_queued,\n sum(throttled_count) AS throttled,\n multiIf(running >= limit AND queued > 0, 'queue-limited', queued > 0, 'backlogged', 'healthy') AS status\nFROM queue_metrics\nGROUP BY queue\nORDER BY peak_queued DESC`, + display: { + type: "table", + prettyFormatting: true, + sorting: [{ id: "peak_queued", desc: true }], + }, + }, + "t-trends": { title: "Per-queue trends", query: "", display: { type: "title" } }, + "running-q": { + title: "Running by queue", + query: `SELECT timeBucket() AS t, queue, max(max_running) AS running\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + // Grouped gauge: carry each queue's running across idle buckets (per-group LOCF). + fillGaps: true, + display: { + type: "chart", + chartType: "line", + xAxisColumn: "t", + yAxisColumns: ["running"], + groupByColumn: "queue", + stacked: false, + sortByColumn: null, + sortDirection: "asc", + aggregation: "max", + }, + }, + "queued-q": { + title: "Queue depth (backlog) by queue", + query: `SELECT timeBucket() AS t, queue, max(max_queued) AS queued\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + // Grouped gauge: carry each queue's backlog across idle buckets (per-group LOCF). + fillGaps: true, + display: { + type: "chart", + chartType: "line", + xAxisColumn: "t", + yAxisColumns: ["queued"], + groupByColumn: "queue", + stacked: false, + sortByColumn: null, + sortDirection: "asc", + aggregation: "max", + }, + }, + "throttled-q": { + title: "Throttled buckets by queue", + query: `SELECT timeBucket() AS t, queue, sum(throttled_count) AS throttled\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + // Grouped counter: per-group zero-fill so idle buckets read 0, not a gap. + fillGaps: true, + display: { + type: "chart", + chartType: "bar", + xAxisColumn: "t", + yAxisColumns: ["throttled"], + groupByColumn: "queue", + stacked: true, + sortByColumn: null, + sortDirection: "asc", + aggregation: "sum", + }, + }, + throughput: { + title: "Enqueued vs started", + // Counter states merge per queue, then sum outside: a single merge across queues + // mixes unrelated odometers and returns wrong totals. + query: `SELECT t, sum(enq) AS enqueued, sum(st) AS started\nFROM (\n SELECT timeBucket() AS t, queue,\n deltaSumTimestampMerge(enqueue_delta) AS enq,\n deltaSumTimestampMerge(started_delta) AS st\n FROM queue_metrics\n GROUP BY t, queue\n)\nGROUP BY t\nORDER BY t`, + display: { + type: "chart", + chartType: "line", + xAxisColumn: "t", + yAxisColumns: ["enqueued", "started"], + groupByColumn: null, + stacked: false, + sortByColumn: null, + sortDirection: "asc", + aggregation: "sum", + }, + }, + "wait-pct": { + title: "Scheduling delay p50/p95/p99 (ms)", + query: `SELECT timeBucket() AS t,\n round(quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[1]) AS p50,\n round(quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n round(quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[4]) AS p99\nFROM env_metrics\nGROUP BY t\nORDER BY t`, + display: { + type: "chart", + chartType: "line", + xAxisColumn: "t", + yAxisColumns: ["p50", "p95", "p99"], + groupByColumn: null, + stacked: false, + sortByColumn: null, + sortDirection: "asc", + aggregation: "max", + }, + }, + }, + }, +}; + +const builtInDashboards: BuiltInDashboard[] = [overviewDashboard, llmDashboard, queuesDashboard]; export function builtInDashboardList(): BuiltInDashboard[] { return builtInDashboards; diff --git a/apps/webapp/app/presenters/v3/MetricDashboardPresenter.server.ts b/apps/webapp/app/presenters/v3/MetricDashboardPresenter.server.ts index df43864b53a..0b84e971b2f 100644 --- a/apps/webapp/app/presenters/v3/MetricDashboardPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/MetricDashboardPresenter.server.ts @@ -37,6 +37,9 @@ export const Widget = z.object({ title: z.string(), query: z.string().default(""), display: QueryWidgetConfig, + // Opt into server-side gap fill (carry-forward for gauges, zero-fill for counters). + // Top-level rather than in `display` because display config is client-only and never reaches the query POST. + fillGaps: z.boolean().optional(), }); export type Widget = z.infer<typeof Widget>; diff --git a/apps/webapp/app/presenters/v3/QueueAllocationPresenter.server.ts b/apps/webapp/app/presenters/v3/QueueAllocationPresenter.server.ts new file mode 100644 index 00000000000..3cb98128de7 --- /dev/null +++ b/apps/webapp/app/presenters/v3/QueueAllocationPresenter.server.ts @@ -0,0 +1,56 @@ +import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { sqlDatabaseSchema } from "~/db.server"; +import { BasePresenter } from "./basePresenter.server"; + +export type QueueAllocation = { + /** Number of V2 queues in the environment. */ + totalQueues: number; + /** Sum of explicit per-queue limits, each clamped to the env limit. */ + allocated: number; + /** Queues with no explicit limit (they float up to the env limit). */ + unlimitedCount: number; +}; + +/** + * Environment-wide allocation totals for the queues page summary tiles. + * + * The page only needs the aggregate `allocated` value (sum of each queue's + * explicit limit clamped to the env limit), so this computes it in a single + * Postgres aggregate over ALL V2 queues in the environment — no row cap and no + * Redis lookups. + */ +export class QueueAllocationPresenter extends BasePresenter { + public async call({ + environment, + }: { + environment: AuthenticatedEnvironment; + }): Promise<QueueAllocation> { + const envLimit = environment.maximumConcurrencyLimit; + + const [row] = await this._replica.$queryRaw< + { + totalQueues: number; + allocated: number; + unlimitedCount: number; + }[] + >` + SELECT + COUNT(*)::int AS "totalQueues", + COUNT(*) FILTER (WHERE "concurrencyLimit" IS NULL)::int AS "unlimitedCount", + COALESCE( + SUM(LEAST("concurrencyLimit", ${envLimit})) + FILTER (WHERE "concurrencyLimit" IS NOT NULL), + 0 + )::int AS "allocated" + FROM ${sqlDatabaseSchema}."TaskQueue" + WHERE "runtimeEnvironmentId" = ${environment.id} + AND "version" = 'V2' + `; + + return { + totalQueues: row?.totalQueues ?? 0, + allocated: row?.allocated ?? 0, + unlimitedCount: row?.unlimitedCount ?? 0, + }; + } +} diff --git a/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts b/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts index 024a1342b0a..6de35f2d45d 100644 --- a/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts @@ -3,6 +3,8 @@ import type { Prisma } from "@trigger.dev/database"; import { TaskQueueType } from "@trigger.dev/database"; import { type PrismaClientOrTransaction } from "~/db.server"; import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; +import { logger } from "~/services/logger.server"; import { determineEngineVersion } from "~/v3/engineVersion.server"; import { engine } from "~/v3/runEngine.server"; import { BasePresenter } from "./basePresenter.server"; @@ -13,6 +15,12 @@ type QueueListEngine = Pick<RunEngine, "lengthOfQueues" | "currentConcurrencyOfQ export const QUEUE_LIST_DEFAULT_ITEMS_PER_PAGE = 25; const MAX_ITEMS_PER_PAGE = 100; +export type QueueListSort = "busiest" | "queued" | "name"; + +/** Ranking reads recent aggregated gauges, so ordering is a stable snapshot, not a live sort. */ +export const QUEUE_RANKING_WINDOW_MINUTES = 15; +const MAX_RANKED_QUEUES = 5000; + const typeToDBQueueType: Record<"task" | "custom", TaskQueueType> = { task: TaskQueueType.VIRTUAL, custom: TaskQueueType.NAMED, @@ -26,10 +34,47 @@ const queueListSelect = { concurrencyLimitBase: true, concurrencyLimitOverriddenAt: true, concurrencyLimitOverriddenBy: true, + concurrencyLimitOverridePercent: true, type: true, paused: true, } satisfies Prisma.TaskQueueSelect; +type QueueListRow = Prisma.TaskQueueGetPayload<{ select: typeof queueListSelect }>; + +// The percent source-of-truth for percent-based overrides isn't part of the shared `QueueItem` +// schema (that's a public contract), so we surface it as an extra field on the list item. +type QueueListItem = ReturnType<typeof toQueueItem> & { + concurrencyLimitOverridePercent: number | null; +}; + +type QueueListPagination = + | { mode: "filtered"; currentPage: number; hasMore: boolean } + | { mode: "unfiltered"; currentPage: number; totalPages: number; count: number }; + +// The `?: undefined` markers keep every key reachable across the union, so consumers +// can destructure before narrowing on `success`. +export type QueueListResult = + | { + success: false; + code: string; + totalQueues: number; + hasFilters: boolean; + queues?: undefined; + pagination?: undefined; + } + | { + success: true; + queues: QueueListItem[]; + pagination: QueueListPagination; + totalQueues?: number; + hasFilters: boolean; + code?: undefined; + }; + +function formatClickhouseDateTime(date: Date): string { + return date.toISOString().slice(0, 19).replace("T", " "); +} + function buildQueueListWhere( environmentId: string, query: string | undefined, @@ -70,13 +115,15 @@ export class QueueListPresenter extends BasePresenter { query, page, type, + sort = "name", }: { environment: AuthenticatedEnvironment; query?: string; page: number; perPage?: number; type?: "task" | "custom"; - }) { + sort?: QueueListSort; + }): Promise<QueueListResult> { const hasFilters = Boolean(query?.trim()) || type !== undefined; const engineVersion = await determineEngineVersion({ environment }); @@ -110,6 +157,18 @@ export class QueueListPresenter extends BasePresenter { }; } + if (sort !== "name") { + // Ranking is additive: any failure or unsupported input falls back to name order. + try { + const ranked = await this.getRankedQueues(environment, query, page, type, sort); + if (ranked) { + return ranked; + } + } catch (error) { + logger.warn("Queue ranking unavailable, falling back to name order", { error }); + } + } + if (hasFilters) { const { queues, hasMore } = await this.getFilteredQueues(environment, query, page, type); @@ -143,6 +202,132 @@ export class QueueListPresenter extends BasePresenter { }; } + /** + * ClickHouse ranks queues by recent activity and returns the requested page of names; + * queues with no recent metrics follow in name order. Null when ranking does not apply. + */ + private async getRankedQueues( + environment: AuthenticatedEnvironment, + query: string | undefined, + page: number, + type: "task" | "custom" | undefined, + sort: Exclude<QueueListSort, "name"> + ) { + if (type !== undefined) { + return null; + } + + const clickhouse = await clickhouseFactory.getClickhouseForOrganization( + environment.organizationId, + "queueMetrics" + ); + + // The window start is aligned to the minute so repeated page loads produce identical + // query text and can share ClickHouse query-cache entries. + const windowStartMs = + Math.floor((Date.now() - QUEUE_RANKING_WINDOW_MINUTES * 60 * 1000) / 60_000) * 60_000; + const rankingArgs = { + organizationId: environment.organizationId, + projectId: environment.projectId, + environmentId: environment.id, + startTime: formatClickhouseDateTime(new Date(windowStartMs)), + nameContains: query?.trim() ?? "", + }; + + const offset = (page - 1) * this.perPage; + + // One scan returns the page and the total ranked count (window function). + const [pageError, pageRows] = await clickhouse.queueMetrics.ranking({ + ...rankingArgs, + byQueuedOnly: sort === "queued" ? 1 : 0, + limit: this.perPage, + offset, + }); + if (pageError) { + throw pageError; + } + + let ranked = pageRows?.[0]?.ranked_total ?? 0; + if (ranked === 0 && offset > 0) { + // Empty page past the ranked head: fetch the count alone for the tail slot math. + const [countError, countRows] = await clickhouse.queueMetrics.rankingCount(rankingArgs); + if (countError) { + throw countError; + } + ranked = countRows?.[0]?.ranked ?? 0; + } + if (ranked > MAX_RANKED_QUEUES) { + return null; + } + + const where = buildQueueListWhere(environment.id, query, type); + const totalQueues = await this._replica.taskQueue.count({ where }); + + let rankedPageQueues: QueueListRow[] = []; + if ((pageRows?.length ?? 0) > 0) { + const rankedNames = (pageRows ?? []).map((row) => row.queue_name); + rankedPageQueues = await this.findQueuesByNames(where, rankedNames); + } + + // Tail of the page: name-ordered queues that have no recent metrics. Slot math uses the + // ClickHouse counts so pages never overlap, even if some ranked names no longer exist. + const rankedSlots = Math.min(Math.max(ranked - offset, 0), this.perPage); + const tailNeeded = this.perPage - rankedSlots; + let tailQueues: QueueListRow[] = []; + if (tailNeeded > 0) { + let excludedNames: string[] = []; + if (ranked > 0) { + const [allError, allRows] = await clickhouse.queueMetrics.rankingNames({ + ...rankingArgs, + limit: MAX_RANKED_QUEUES, + }); + if (allError) { + throw allError; + } + excludedNames = (allRows ?? []).map((row) => row.queue_name); + } + // AND keeps the search's name filter intact alongside the exclusion (a spread + // would overwrite one name condition with the other). + tailQueues = await this._replica.taskQueue.findMany({ + where: { AND: [where, { name: { notIn: excludedNames } }] }, + select: queueListSelect, + orderBy: { + orderableName: "asc", + }, + skip: Math.max(0, offset - ranked), + take: tailNeeded, + }); + } + + return { + success: true as const, + queues: await this.enrichQueues(environment, [...rankedPageQueues, ...tailQueues]), + pagination: { + mode: "unfiltered" as const, + currentPage: page, + totalPages: Math.max(1, Math.ceil(totalQueues / this.perPage)), + count: totalQueues, + }, + totalQueues, + hasFilters: Boolean(query?.trim()) || type !== undefined, + }; + } + + private async findQueuesByNames( + where: Prisma.TaskQueueWhereInput, + names: string[] + ): Promise<QueueListRow[]> { + if (names.length === 0) { + return []; + } + const queues = await this._replica.taskQueue.findMany({ + where: { AND: [where, { name: { in: names } }] }, + select: queueListSelect, + }); + const byName = new Map(queues.map((queue) => [queue.name, queue])); + return names.flatMap((name) => byName.get(name) ?? []); + } + private async getFilteredQueues( environment: AuthenticatedEnvironment, query: string | undefined, @@ -195,10 +380,11 @@ export class QueueListPresenter extends BasePresenter { concurrencyLimitBase: number | null; concurrencyLimitOverriddenAt: Date | null; concurrencyLimitOverriddenBy: string | null; + concurrencyLimitOverridePercent: Prisma.Decimal | null; type: TaskQueueType; paused: boolean; }[] - ) { + ): Promise<QueueListItem[]> { const [queuedByQueue, runningByQueue] = await Promise.all([ this.engineClient.lengthOfQueues( environment, @@ -221,8 +407,8 @@ export class QueueListPresenter extends BasePresenter { const overriddenByMap = new Map(overriddenByUsers.map((u) => [u.id, u])); - return queues.map((queue) => - toQueueItem({ + return queues.map((queue) => ({ + ...toQueueItem({ friendlyId: queue.friendlyId, name: queue.name, type: queue.type, @@ -235,7 +421,12 @@ export class QueueListPresenter extends BasePresenter { ? (overriddenByMap.get(queue.concurrencyLimitOverriddenBy) ?? null) : null, paused: queue.paused, - }) - ); + }), + // Prisma returns Decimal; the client only needs a plain number (null for absolute overrides). + concurrencyLimitOverridePercent: + queue.concurrencyLimitOverridePercent !== null + ? Number(queue.concurrencyLimitOverridePercent) + : null, + })); } } diff --git a/apps/webapp/app/presenters/v3/QueueMetricsPresenter.server.ts b/apps/webapp/app/presenters/v3/QueueMetricsPresenter.server.ts new file mode 100644 index 00000000000..c5c4c082850 --- /dev/null +++ b/apps/webapp/app/presenters/v3/QueueMetricsPresenter.server.ts @@ -0,0 +1,145 @@ +import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; +import { logger } from "~/services/logger.server"; +import { computeSparklineGrid } from "~/v3/queueSparklineGrid"; + +export type QueueListMetric = { + p50WaitMs: number | null; + p95WaitMs: number | null; + peakQueued: number; + /** Times this queue was throttled (running at limit with a backlog) over the window. */ + throttledTotal: number; + /** Equal-width buckets, oldest first, carry-forward filled across idle gaps. */ + depthSparkline: number[]; + /** + * Throttled count per bucket, aligned 1:1 with `depthSparkline`. Not carry-forward filled — + * a bucket is non-zero only when throttling actually happened in it, so callers can tint + * exactly those bars. + */ + throttledSparkline: number[]; +}; + +export type QueueListMetrics = { + bucketStartMs: number; + bucketIntervalMs: number; + byQueue: Map<string, QueueListMetric>; +}; + +function formatClickhouseDateTime(date: Date): string { + return date.toISOString().slice(0, 19).replace("T", " "); +} + +function finiteOrNull(value: number): number | null { + return Number.isFinite(value) ? value : null; +} + +export class QueueMetricsPresenter { + /** + * Per-queue metrics over a time range for a fixed set of queues (the visible list page), + * scoped to one ClickHouse query window so cost is independent of total queue count. + * Degrades to an empty map if ClickHouse is unavailable so the live list still renders. + */ + public async getQueueListMetrics({ + environment, + queueNames, + from, + to, + }: { + environment: AuthenticatedEnvironment; + queueNames: string[]; + from: Date; + to: Date; + }): Promise<QueueListMetrics> { + const grid = computeSparklineGrid(from, to); + const { bucketSeconds, bucketIntervalMs, bucketStartMs } = grid; + const numBuckets = grid.bucketCount; + + const empty: QueueListMetrics = { + bucketStartMs, + bucketIntervalMs, + byQueue: new Map(), + }; + + if (queueNames.length === 0) { + return empty; + } + + try { + const clickhouse = await clickhouseFactory.getClickhouseForOrganization( + environment.organizationId, + "queueMetrics" + ); + + const endMs = grid.endMs; + const ids = { + organizationId: environment.organizationId, + projectId: environment.projectId, + environmentId: environment.id, + queueNames, + startTime: formatClickhouseDateTime(new Date(bucketStartMs)), + endTime: formatClickhouseDateTime(new Date(endMs)), + }; + + const [summaryResult, sparklineResult] = await Promise.all([ + clickhouse.queueMetrics.listSummary(ids), + clickhouse.queueMetrics.depthSparklines({ ...ids, bucketSeconds }), + ]); + + const [summaryError, summaryRows] = summaryResult; + const [sparklineError, sparklineRows] = sparklineResult; + + if (summaryError || sparklineError) { + logger.warn("QueueMetricsPresenter: clickhouse query failed", { + summaryError: summaryError?.message, + sparklineError: sparklineError?.message, + }); + return empty; + } + + // Bucket -> depth + throttled per queue, mapped onto the aligned grid. Depth is + // forward-filled below; throttled is not (only real per-bucket counts tint bars). + const bucketsByQueue = new Map<string, Map<number, { depth: number; throttled: number }>>(); + for (const row of sparklineRows ?? []) { + const bucketMs = Date.parse(row.bucket.replace(" ", "T") + "Z"); + if (Number.isNaN(bucketMs)) continue; + const index = Math.round((bucketMs - bucketStartMs) / bucketIntervalMs); + if (index < 0 || index >= numBuckets) continue; + let byIndex = bucketsByQueue.get(row.queue_name); + if (!byIndex) { + byIndex = new Map(); + bucketsByQueue.set(row.queue_name, byIndex); + } + byIndex.set(index, { depth: row.depth, throttled: row.throttled }); + } + + const byQueue = new Map<string, QueueListMetric>(); + for (const row of summaryRows ?? []) { + const byIndex = bucketsByQueue.get(row.queue_name); + const sparkline: number[] = new Array(numBuckets); + const throttledSparkline: number[] = new Array(numBuckets); + let last = 0; + for (let i = 0; i < numBuckets; i++) { + const bucket = byIndex?.get(i); + if (bucket !== undefined) last = bucket.depth; + sparkline[i] = last; + throttledSparkline[i] = bucket?.throttled ?? 0; + } + byQueue.set(row.queue_name, { + p50WaitMs: finiteOrNull(row.p50_wait_ms), + p95WaitMs: finiteOrNull(row.p95_wait_ms), + peakQueued: row.peak_queued, + throttledTotal: row.throttled_count, + depthSparkline: sparkline, + throttledSparkline, + }); + } + + return { bucketStartMs, bucketIntervalMs, byQueue }; + } catch (error) { + logger.warn("QueueMetricsPresenter: failed to load queue metrics", { + error: error instanceof Error ? error.message : String(error), + }); + return empty; + } + } +} diff --git a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts index b446dfaf626..08e0d751e39 100644 --- a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts @@ -106,18 +106,28 @@ export class QueueRetrievePresenter extends BasePresenter { // Transform queues to include running and queued counts return { success: true as const, - queue: toQueueItem({ - friendlyId: queue.friendlyId, - name: queue.name, - type: queue.type, - running: results[1]?.[queue.name] ?? 0, - queued: results[0]?.[queue.name] ?? 0, - concurrencyLimit: queue.concurrencyLimit ?? null, - concurrencyLimitBase: queue.concurrencyLimitBase ?? null, - concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt ?? null, - concurrencyLimitOverriddenBy: queue.concurrencyLimitOverriddenBy ?? null, - paused: queue.paused, - }), + queue: { + ...toQueueItem({ + friendlyId: queue.friendlyId, + name: queue.name, + type: queue.type, + running: results[1]?.[queue.name] ?? 0, + queued: results[0]?.[queue.name] ?? 0, + concurrencyLimit: queue.concurrencyLimit ?? null, + concurrencyLimitBase: queue.concurrencyLimitBase ?? null, + concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt ?? null, + concurrencyLimitOverriddenBy: queue.concurrencyLimitOverriddenBy ?? null, + paused: queue.paused, + }), + // The percent source-of-truth for percent-based overrides isn't part of the shared + // `QueueItem` schema (that's a public contract), so we surface it as an extra field on + // the returned queue — mirroring QueueListPresenter. Prisma returns Decimal; the client + // only needs a plain number (null for absolute overrides). + concurrencyLimitOverridePercent: + queue.concurrencyLimitOverridePercent !== null + ? Number(queue.concurrencyLimitOverridePercent) + : null, + }, }; } } diff --git a/apps/webapp/app/presenters/v3/RunQueueMetricsPresenter.server.ts b/apps/webapp/app/presenters/v3/RunQueueMetricsPresenter.server.ts new file mode 100644 index 00000000000..0defa6178ab --- /dev/null +++ b/apps/webapp/app/presenters/v3/RunQueueMetricsPresenter.server.ts @@ -0,0 +1,174 @@ +import { $replica } from "~/db.server"; +import { findProjectBySlug } from "~/models/project.server"; +import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; +import { logger } from "~/services/logger.server"; +import { canAccessQueueMetricsUi } from "~/v3/canAccessQueueMetricsUi.server"; +import { engine } from "~/v3/runEngine.server"; + +export type RunQueueWaiting = { + queued: number; + running: number; + concurrencyLimit: number | null; + delayP50Ms: number | null; + delayP95Ms: number | null; + loadedAt: number; + ids: { organizationId: string; projectId: string; environmentId: string }; + concurrencyKey: { + key: string; + queued: number; + running: number; + oldestWaitMs: number | null; + } | null; +}; + +export type RunQueueMetrics = { + queueFriendlyId: string | null; + queueName: string; + paused: boolean; + waiting: RunQueueWaiting | null; +}; + +// PENDING renders as "Queued" in the dashboard. +const WAITING_STATUSES = new Set(["PENDING", "DELAYED", "PENDING_VERSION"]); + +const DELAY_WINDOW_MS = 60 * 60 * 1000; +// Window bounds snap to this grid so repeated span opens share ClickHouse cache entries. +const DELAY_GRID_MS = 5 * 60 * 1000; + +/** + * Queue context for the run inspector: the queue's friendlyId for linking plus, for runs + * waiting to start, live counts and recent delay percentiles. Null when flag off. + */ +export async function resolveRunQueueMetrics(options: { + userId: string; + organizationSlug: string; + projectParam: string; + envParam: string; + run: { + environmentId: string; + status: string; + engine: string; + queue: { name: string; concurrencyKey?: string | null }; + }; +}): Promise<RunQueueMetrics | null> { + const { userId, organizationSlug, projectParam, envParam, run } = options; + + try { + if (!(await canAccessQueueMetricsUi({ userId, organizationSlug }))) { + return null; + } + + const taskQueue = await $replica.taskQueue.findFirst({ + where: { runtimeEnvironmentId: run.environmentId, name: run.queue.name }, + select: { friendlyId: true, concurrencyLimit: true, paused: true }, + }); + + const base: RunQueueMetrics = { + queueFriendlyId: taskQueue?.friendlyId ?? null, + queueName: run.queue.name, + paused: taskQueue?.paused ?? false, + waiting: null, + }; + + // Live reads go to the Run Engine 2.0 run-queue, so V1 runs get the link only. + if (run.engine !== "V2" || !WAITING_STATUSES.has(run.status)) { + return base; + } + + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + if (!project) return base; + const environment = await findEnvironmentBySlug(project.id, envParam, userId); + if (!environment) return base; + + const queueName = run.queue.name; + const ck = run.queue.concurrencyKey ?? undefined; + + const [queued, concurrency, ckQueued, ckRunning, ckOldest, delays] = await Promise.all([ + engine.lengthOfQueue(environment, queueName), + engine.currentConcurrencyOfQueues(environment, [queueName]), + ck ? engine.lengthOfQueue(environment, queueName, ck) : null, + ck ? engine.currentConcurrencyOfQueue(environment, queueName, ck) : null, + ck ? engine.oldestMessageInQueue(environment, queueName, ck) : undefined, + delaySummary({ + organizationId: project.organizationId, + projectId: project.id, + environmentId: environment.id, + queueName, + }), + ]); + + const loadedAt = Date.now(); + + return { + ...base, + waiting: { + queued, + running: concurrency?.[queueName] ?? 0, + concurrencyLimit: taskQueue?.concurrencyLimit ?? null, + delayP50Ms: delays.p50, + delayP95Ms: delays.p95, + loadedAt, + ids: { + organizationId: project.organizationId, + projectId: project.id, + environmentId: environment.id, + }, + concurrencyKey: ck + ? { + key: ck, + queued: ckQueued ?? 0, + running: ckRunning ?? 0, + oldestWaitMs: typeof ckOldest === "number" ? Math.max(0, loadedAt - ckOldest) : null, + } + : null, + }, + }; + } catch (error) { + logger.warn("resolveRunQueueMetrics failed", { + error: error instanceof Error ? error.message : String(error), + }); + return null; + } +} + +function formatClickhouseDateTime(date: Date): string { + return date.toISOString().slice(0, 19).replace("T", " "); +} + +async function delaySummary(input: { + organizationId: string; + projectId: string; + environmentId: string; + queueName: string; +}): Promise<{ p50: number | null; p95: number | null }> { + try { + const clickhouse = await clickhouseFactory.getClickhouseForOrganization( + input.organizationId, + "queueMetrics" + ); + + const endMs = Math.ceil(Date.now() / DELAY_GRID_MS) * DELAY_GRID_MS; + const startMs = endMs - DELAY_WINDOW_MS; + + const [error, rows] = await clickhouse.queueMetrics.listSummary({ + organizationId: input.organizationId, + projectId: input.projectId, + environmentId: input.environmentId, + queueNames: [input.queueName], + startTime: formatClickhouseDateTime(new Date(startMs)), + endTime: formatClickhouseDateTime(new Date(endMs)), + }); + + if (error) return { p50: null, p95: null }; + const row = rows?.[0]; + if (!row) return { p50: null, p95: null }; + + return { + p50: Number.isFinite(row.p50_wait_ms) ? row.p50_wait_ms : null, + p95: Number.isFinite(row.p95_wait_ms) ? row.p95_wait_ms : null, + }; + } catch { + return { p50: null, p95: null }; + } +} diff --git a/apps/webapp/app/presenters/v3/reports/ReportPresenter.server.ts b/apps/webapp/app/presenters/v3/reports/ReportPresenter.server.ts new file mode 100644 index 00000000000..1393fd611df --- /dev/null +++ b/apps/webapp/app/presenters/v3/reports/ReportPresenter.server.ts @@ -0,0 +1,49 @@ +/** + * Thin orchestrator: load -> interpret -> return the generic ReportViewModel. No SQL or render + * here — data access lives in each report's `load`, meaning in `interpret`, presentation in the + * renderers, and the catalog of reports in `report-registry.ts`. + * + * `call` takes a resolved AuthenticatedEnvironment and is transport-independent (Seam B, §7): + * any future surface (MCP Resource, etc.) is just another caller of this same method. + */ + +import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { REPORT_REGISTRY } from "./report-registry"; +import { type ReportViewModel } from "./report-view-model"; + +const DEFAULT_PERIOD = "1h"; + +/** + * Single-flight: collapse concurrent identical requests (same report/env/period) into one + * computation. A report fires up to ~9 ClickHouse queries and MCP/CLI clients easily call it + * several times at once — without this, N callers each launch the full query set and pile onto + * the per-project query-concurrency limit. Keyed per (key, env, period); entry drops on settle. + */ +const inFlight = new Map<string, Promise<ReportViewModel | undefined>>(); + +export class ReportPresenter { + async call({ + environment, + key, + period = DEFAULT_PERIOD, + }: { + environment: AuthenticatedEnvironment; + key: string; + period?: string; + }): Promise<ReportViewModel | undefined> { + const loader = REPORT_REGISTRY[key]; + if (!loader) return undefined; + + const flightKey = `${key} ${environment.id} ${period}`; + const existing = inFlight.get(flightKey); + if (existing) return existing; + + const promise = (async () => { + const input = await loader.load(environment, period); + return loader.interpret(input); + })().finally(() => inFlight.delete(flightKey)); + + inFlight.set(flightKey, promise); + return promise; + } +} diff --git a/apps/webapp/app/presenters/v3/reports/health/execution.ts b/apps/webapp/app/presenters/v3/reports/health/execution.ts new file mode 100644 index 00000000000..12d19cef603 --- /dev/null +++ b/apps/webapp/app/presenters/v3/reports/health/execution.ts @@ -0,0 +1,55 @@ +/** + * EXECUTION analyzer: "are the runs that DO start completing OK?" — failures + durations only. + * Its "read:" line answers whether the flow problem is a code problem. + */ + +import { isOk, maxSeverity, type Finding, type Metric } from "../report-view-model"; +import { HEALTH_THRESHOLDS, metricById, type HealthInput } from "./health-core"; + +export const EXECUTION_METRIC_IDS = ["failures", "dur_p95"]; + +export function interpretExecution(metrics: Metric[], input: HealthInput): Finding { + const exec = EXECUTION_METRIC_IDS.map((id) => metricById(metrics, id)); + const severity = maxSeverity(...exec.map((m) => m.severity)); + const failures = metricById(metrics, "failures"); + + if (isOk(severity)) { + return { type: "execution", severity, reason: "healthy", metricIds: EXECUTION_METRIC_IDS }; + } + + const reason = !isOk(failures.severity) ? "failures_up" : "slow_runs"; + const recommendation = + reason === "failures_up" + ? { code: "review_failing_tasks", link: "runs_failed" } + : { code: "review_slow_runs", link: "runs" }; + + // Lazy attribution when it owns >= minShare of failures. + let attribution: Finding["attribution"]; + const fb = input.failureBreakdown; + if (fb && fb.share >= HEALTH_THRESHOLDS.attribution.minShare) { + attribution = { dim: "task", key: fb.task, share: fb.share, of: "failures" }; + } + + return { + type: "execution", + severity, + reason, + metricIds: EXECUTION_METRIC_IDS, + recommendation, + attribution, + }; +} + +/** Flow causes that provably CAN'T be user code, so a healthy execution reads "not a code problem". + * dequeue_stall is platform-side (capacity free but nothing dequeuing). Trigger spike/surge are + * excluded: a code path fanning out task.trigger can BE the cause, so we only state the runs that + * execute are fine — never the global "not a code problem". */ +const NOT_A_CODE_PROBLEM_CAUSES = new Set(["dequeue_stall"]); + +export function buildExecutionRead(execution: Finding, flow: Finding): string { + if (execution.reason === "unknown") return "data_stale"; + if (isOk(execution.severity)) { + return NOT_A_CODE_PROBLEM_CAUSES.has(flow.reason) ? "not_a_code_problem" : "runs_are_fine"; + } + return "failures_elevated"; +} diff --git a/apps/webapp/app/presenters/v3/reports/health/flow.ts b/apps/webapp/app/presenters/v3/reports/health/flow.ts new file mode 100644 index 00000000000..a531a4aeb11 --- /dev/null +++ b/apps/webapp/app/presenters/v3/reports/health/flow.ts @@ -0,0 +1,304 @@ +/** + * FLOW analyzer: "is work flowing, and if not, WHY?" Diagnosed by a CAUSE TREE — `flow.reason` + * names why work is backing up (env_limit_saturation, dequeue_stall, …), selected from flow's + * evidence, falling back to v1 symptom reasons when no discriminator fires. Evidence quantities + * carry no severity of their own — they only select the cause. + */ + +import { + anomalyWindow, + isOk, + maxSeverity, + type Exclusion, + type Finding, + type Metric, + type Observation, + type Recommendation, + type Severity, +} from "../report-view-model"; +import { + HEALTH_THRESHOLDS, + isPendingIncreasing, + mean, + metricById, + type HealthInput, +} from "./health-core"; + +export const FLOW_METRIC_IDS = ["start_latency_p95", "pending", "throughput"]; + +/** One row of the declarative cause table — everything a cause defines about itself. */ +type CauseSpec = { + reason: string; + metricIds: string[]; // real metric rows, causal order + drivingMetricId: string; // series for the anomaly window + annotationCode?: string; // set on the driving metric + exclusions: Exclusion[]; // ruled-out causes ("not your code") + observations: Observation[]; // supporting facts ("runs are completing at ~X/min") + recommendation: Recommendation; + usesAttribution: boolean; // append the worst-queue attribution line +}; + +export function interpretFlow(metrics: Metric[], input: HealthInput): Finding { + const t = HEALTH_THRESHOLDS.flowCause; + const ev = input.flowEvidence; + const flowMetrics = FLOW_METRIC_IDS.map((id) => metricById(metrics, id)); + const severity = maxSeverity(...flowMetrics.map((m) => m.severity)); + + if (isOk(severity)) { + return { type: "flow", severity, reason: "healthy", metricIds: FLOW_METRIC_IDS }; + } + + // Discriminators (evidence only, no own severity). + const pendingIncreasing = isPendingIncreasing(input.pending.series); + const latencyElevated = !isOk(metricById(metrics, "start_latency_p95").severity); + // Concurrency causes need real running-capacity evidence — without it runningShare is a + // meaningless 0 and would falsely select dequeue_stall on the snapshot path (#1). + const hasConcurrencyEvidence = ev.envLimit > 0 && ev.runningSeries.length > 0; + const runningShare = hasConcurrencyEvidence ? mean(ev.runningSeries) / ev.envLimit : 1; + const pinnedShare = hasConcurrencyEvidence + ? ev.runningSeries.filter((r) => r >= t.pinnedLevel * ev.envLimit).length / + ev.runningSeries.length + : 0; + const pinned = pinnedShare >= t.pinnedShare; + const hasTriggerBaseline = input.throughput.normalTriggeredPerMin > 0; + const triggeredMult = hasTriggerBaseline + ? input.throughput.triggeredPerMin / input.throughput.normalTriggeredPerMin + : 0; + // No baseline: a multiplier can't be computed, so an absolute rate selects "new volume". + const triggerSurge = !hasTriggerBaseline && input.throughput.triggeredPerMin >= t.surgePerMin; + + const donePerMin = input.throughput.donePerMin; + const net = donePerMin - input.throughput.triggeredPerMin; + // Exclusions must be PROVEN, not assumed. "not your code" needs healthy execution; "limits + // aren't the bottleneck" needs no env-pin AND no queue throttling; the workers/spike ones + // state a measured fact (rate) rather than a global "everything's fine" claim. + const executionHealthy = + isOk(metricById(metrics, "failures").severity) && isOk(metricById(metrics, "dur_p95").severity); + const queueThrottled = ev.throttledShare >= t.throttledShare; + const configHealthy = !pinned && !queueThrottled; + // A trigger spike/surge is only the CAUSE of a backup when work is actually piling up: + // completions falling behind (net < 0) AND the backlog trending up. Without this a spike that + // drains fine would still be blamed while its read says "queue fills faster than it drains". + const triggerBacklog = net < 0 && pendingIncreasing; + + // First discriminator that fires wins (fixed priority). dequeue_stall is a last-resort + // "it's on our side" cause — so a known config bottleneck (queue throttling) must rule it + // out first, else throttled-but-idle-capacity is misread as a platform stall (#1). + let spec: CauseSpec; + if ( + hasConcurrencyEvidence && + !queueThrottled && + runningShare < t.stallRunningShare && + pendingIncreasing && + latencyElevated + ) { + spec = { + reason: "dequeue_stall", + metricIds: ["concurrency", "pending", "start_latency_p95"], + drivingMetricId: "concurrency", + annotationCode: "idle_share", + exclusions: [ + ...(executionHealthy ? [{ code: "not_your_code" }] : []), + ...(configHealthy ? [{ code: "not_your_config" }] : []), + ], + observations: [], + recommendation: { code: "check_platform_status", link: "status" }, + usesAttribution: false, + }; + } else if (pinned && pendingIncreasing) { + spec = { + reason: "env_limit_saturation", + metricIds: ["concurrency", "pending", "start_latency_p95"], + drivingMetricId: "concurrency", + annotationCode: "pinned_minutes", + exclusions: [], + // States a measured fact (runs ARE completing at {rate}/min) — evidence the workers aren't + // dead. An observation, not an exclusion: it doesn't claim it's the limit, nor "keeps pace". + observations: + donePerMin > 0 ? [{ code: "not_workers_platform", evidence: { donePerMin } }] : [], + recommendation: { code: "raise_env_limit", link: "concurrency" }, + usesAttribution: true, + }; + } else if (ev.throttledShare >= t.throttledShare && !pinned) { + spec = { + reason: "queue_limit_throttling", + metricIds: ["throttled", "pending"], + drivingMetricId: "throttled", + annotationCode: "throttled_minutes", + // Justified: we're in the not-pinned branch, so the env limit isn't the bottleneck. + exclusions: [{ code: "not_env_limit" }], + observations: [], + recommendation: { code: "raise_queue_limit", link: "queue" }, + usesAttribution: true, + }; + } else if (triggeredMult >= t.spikeMult && triggerBacklog) { + spec = { + reason: "trigger_spike", + metricIds: ["triggered", "pending", "start_latency_p95"], + drivingMetricId: "triggered", + annotationCode: "spike_mult", + exclusions: [], + // Only the proven fact — the runs that DO start execute fine. An observation, NOT the + // exclusion "not your code": healthy execution doesn't prove the code isn't the one + // triggering the flood (e.g. a deploy that fans out task.trigger in a loop). + observations: executionHealthy ? [{ code: "execution_healthy" }] : [], + recommendation: { code: "review_trigger_source", link: "runs" }, + usesAttribution: false, + }; + } else if (triggerSurge && triggerBacklog) { + spec = { + reason: "trigger_surge", + metricIds: ["triggered", "pending", "start_latency_p95"], + drivingMetricId: "triggered", + annotationCode: "surge_rate", + exclusions: [], + // Same as a spike: only the proven fact, without ruling out the trigger-producing code. + observations: executionHealthy ? [{ code: "execution_healthy" }] : [], + recommendation: { code: "review_trigger_source", link: "runs" }, + usesAttribution: false, + }; + } else { + // Fallback — v1 symptom reasons by dominant metric. + return fallbackFlow(flowMetrics, severity); + } + + return assembleFlowCause(spec, metrics, input, severity); +} + +/** Build a flow Finding from a cause spec: annotation, anomaly window, attribution, evidence. */ +function assembleFlowCause( + spec: CauseSpec, + metrics: Metric[], + input: HealthInput, + severity: Severity +): Finding { + const t = HEALTH_THRESHOLDS; + const driving = metricById(metrics, spec.drivingMetricId); + + // Anomaly window from the driving series. env_limit_saturation breaches ABOVE + // (concurrency pinned at the limit); dequeue_stall breaches BELOW (capacity idle). + // NOTE: runningSeries is at native env_metrics resolution (not resampled), so the "(last N + // min)" figure assumes those buckets are uniform and cover the resolved window. env_metrics + // are emitted on a fixed cadence, so that holds; a gappy/partial window could skew the minutes. + let aw: Finding["anomalyWindow"]; + if (spec.reason === "env_limit_saturation" || spec.reason === "dequeue_stall") { + const below = spec.reason === "dequeue_stall"; + const threshold = below + ? t.flowCause.stallRunningShare * input.flowEvidence.envLimit + : t.flowCause.pinnedLevel * input.flowEvidence.envLimit; + aw = anomalyWindow(input.flowEvidence.runningSeries, threshold, input.windowMinutes, { below }); + } + + // Annotation on the driving metric (a fact, not an invented number). + if (spec.annotationCode) { + const value = + spec.annotationCode === "pinned_minutes" + ? (aw?.minutes ?? 0) + : spec.annotationCode === "idle_share" + ? // mean running over the window ("N running of {limit}"). + Math.round(mean(input.flowEvidence.runningSeries)) + : spec.annotationCode === "spike_mult" + ? Math.round( + input.throughput.normalTriggeredPerMin > 0 + ? input.throughput.triggeredPerMin / input.throughput.normalTriggeredPerMin + : 0 + ) + : spec.annotationCode === "throttled_minutes" + ? Math.round(input.flowEvidence.throttledShare * input.windowMinutes) + : spec.annotationCode === "surge_rate" + ? Math.round(input.throughput.triggeredPerMin) + : Math.round(driving.value); + driving.annotation = { code: spec.annotationCode, value }; + } + + // Attribution — only when a queue owns >= minShare of the problem. + let attribution: Finding["attribution"]; + const wq = input.flowEvidence.worstQueue; + if (spec.usesAttribution && wq && wq.share >= t.attribution.minShare) { + attribution = { dim: "queue", key: wq.name, share: wq.share, of: "pending" }; + } + + // Append "nothing dead-lettered" ONLY on a measured zero (dlqDelta === 0); null means + // unmeasured (snapshot path) — no observation without evidence. It's a supporting fact, not a + // ruled-out cause, so it joins observations. + const observations = + input.flowEvidence.dlqDelta === 0 + ? [...spec.observations, { code: "nothing_dead_lettered", evidence: { dlq: 0 } }] + : spec.observations; + + return { + type: "flow", + severity, + reason: spec.reason, + metricIds: spec.metricIds, + recommendation: spec.recommendation, + anomalyWindow: aw, + attribution, + exclusions: spec.exclusions, + observations, + }; +} + +/** v1 symptom fallback when no cause discriminator fires. */ +function fallbackFlow(flowMetrics: Metric[], severity: Severity): Finding { + const firstOff = flowMetrics.find((m) => !isOk(m.severity)); + const reason = + firstOff?.id === "start_latency_p95" + ? "start_latency" + : firstOff?.id === "pending" + ? "backlog" + : firstOff?.id === "throughput" + ? "throughput_lag" + : "degraded"; + const recommendation = + reason === "start_latency" + ? { code: "review_start_latency", link: "queue_latency" } + : reason === "backlog" + ? { code: "check_queue_health", link: "queues" } + : reason === "throughput_lag" + ? { code: "check_worker_availability", link: "queues" } + : undefined; + return { + type: "flow", + severity, + reason, + metricIds: FLOW_METRIC_IDS, + recommendation, + }; +} + +// --------------------------------------------------------------------------- +// Flow severity policy + the causal "read:" line. +// --------------------------------------------------------------------------- + +export function applyFlowPolicy( + flow: Finding, + execution: Finding, + isDrainable: boolean, + telemetryStale: boolean +): Finding { + if (flow.severity !== "crit") return flow; + // Downgrade a drainable crit to warn only when execution is fine and telemetry isn't stale. + // Unknown/lagging freshness must NOT block this (it's not a signal that anything's wrong). + const severity: Severity = + isOk(execution.severity) && !telemetryStale && isDrainable ? "warn" : "crit"; + return { ...flow, severity }; +} + +const CAUSE_READS: Record<string, string> = { + dequeue_stall: "capacity_free_not_dequeuing", + env_limit_saturation: "saturation_chain", + queue_limit_throttling: "queue_throttle_chain", + trigger_spike: "spike_chain", + trigger_surge: "surge_chain", +}; + +export function buildFlowRead(flow: Finding, executionOk: boolean, livenessFresh: boolean): string { + if (flow.reason === "unknown") return "data_stale"; // stale-guarded — no causal read + if (isOk(flow.severity)) return "starting_normally"; + if (CAUSE_READS[flow.reason]) return CAUSE_READS[flow.reason]; + // fallback symptoms (v1 logic) + if (executionOk && livenessFresh) return "lag_while_triggering_normal"; + if (!executionOk) return "lag_and_failures"; + return "degraded_generic"; +} diff --git a/apps/webapp/app/presenters/v3/reports/health/health-core.ts b/apps/webapp/app/presenters/v3/reports/health/health-core.ts new file mode 100644 index 00000000000..78fb1d9fbef --- /dev/null +++ b/apps/webapp/app/presenters/v3/reports/health/health-core.ts @@ -0,0 +1,271 @@ +/** + * Health report FOUNDATION shared by the three analyzers (flow / execution / liveness): + * the input shape, tunable thresholds, small helpers, and `buildMetrics` (numbers + + * per-metric severity). No verdict logic here — that lives in the per-analyzer modules. + */ + +import { classifySeverity, delta, type Metric, type Severity } from "../report-view-model"; + +export type HealthInput = { + scope: string; + period: string; + baselineLabel: string; + generatedAt: string; + /** live window length in minutes — for anomaly-window / annotation math. */ + windowMinutes: number; + /** provenance; drives caveat text, not logic. */ + flowSource: "snapshot+runs" | "queue_metrics_v1"; + /** + * now = live env-level depth; normal = 7d baseline (omitted on the snapshot path, which has + * no real 7d pending baseline — so we never mislabel a live-window average as "7d normal"); + * series measured (v2) or estimated (v1). + */ + pending: { now: number; normal?: number; series: number[]; estimated: boolean }; + startLatency: { p95Ms: number; normalP95Ms: number; series: number[] }; + throughput: { donePerMin: number; triggeredPerMin: number; normalTriggeredPerMin: number }; + failures: { rate: number; normalRate: number; series: number[] }; + duration: { p95Ms: number; normalP95Ms: number }; + /** Age of the freshest telemetry (ms). null = no signal to assess -> freshness unknown. */ + liveness: { telemetryAgeMs: number | null }; + /** + * Flow's cause-tree discriminators (no own severity) — from env_metrics rows + one + * queue_metrics GROUP BY. Empty-ish when unavailable (cause tree falls back to v1). + */ + flowEvidence: { + runningSeries: number[]; + envLimit: number; + throttledShare: number; + worstQueue: { name: string; share: number } | null; + /** runs dead-lettered in the window: 0 = measured none, null = unmeasured (snapshot). */ + dlqDelta: number | null; + }; + /** Lazy — loaded only when execution degrades (attribution line). */ + failureBreakdown?: { task: string; share: number; region?: string }; +}; + +/** Tunable defaults — first-guess; tune against prod once wired. */ +export const HEALTH_THRESHOLDS = { + // `floor` = absolute warn/crit used when there's no usable baseline (normal 0/undefined), + // so a spike from a zero baseline (e.g. a never-failing env) isn't classified healthy. + startLatency: { warnMult: 3, critMult: 10, floor: { warn: 30_000, crit: 120_000 } }, + pending: { warnMult: 2, critMult: 10, floor: { warn: 500, crit: 5_000 } }, + failures: { + warnMult: 2, + critMult: 4, + floorRate: 0.005, + floor: { warn: 0.02, crit: 0.05 }, + }, + duration: { warnMult: 1.5, critMult: 3, floor: { warn: 60_000, crit: 600_000 } }, + liveness: { freshMs: 60_000, staleMs: 300_000 }, + flowPolicy: { drainCritMinutes: 60 }, + flowCause: { + stallRunningShare: 0.3, // dequeue_stall: running/limit below this while pending grows + pinnedLevel: 0.95, // a bucket counts as "pinned" when running >= 95% of limit + pinnedShare: 0.5, // env_limit_saturation: >= half the window's buckets pinned + throttledShare: 0.25, // queue_limit_throttling: >= a quarter of the window throttled + spikeMult: 3, // trigger_spike: triggered/min >= 3x the 7d-normal rate + // trigger_surge: with NO usable baseline (normal 0), a multiplier is meaningless, so an + // absolute floor picks the "new volume" cause instead of dropping to the v1 fallback. + surgePerMin: 100, + }, + attribution: { minShare: 0.5 }, // name a queue/task/region only when it owns >= half the problem +}; + +// --------------------------------------------------------------------------- +// Helpers. +// --------------------------------------------------------------------------- + +export const mean = (xs: number[]) => + xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length; + +/** Deterministic "trending up": mean of the last third vs the first third (direction only). */ +export function isPendingIncreasing(series: number[]): boolean { + if (series.length < 2) return false; + const third = Math.max(1, Math.floor(series.length / 3)); + return mean(series.slice(-third)) > mean(series.slice(0, third)); +} + +function multiplierSeverity( + value: number, + normal: number | undefined, + warnMult: number, + critMult: number, + floor?: { warn: number; crit: number } +): Severity { + if (normal === undefined || !Number.isFinite(normal) || normal === 0) { + // No usable baseline: fall back to an absolute floor so a spike isn't a false green. + return floor ? classifySeverity(value, floor) : "ok"; + } + return classifySeverity(value / normal, { warn: warnMult, crit: critMult }); +} + +/** Look up a metric by id; throws if absent (buildMetrics guarantees the standard set exists). */ +export function metricById(metrics: Metric[], id: string): Metric { + const m = metrics.find((x) => x.id === id); + if (!m) throw new Error(`health: missing metric ${id}`); + return m; +} + +// --------------------------------------------------------------------------- +// buildMetrics: numbers + per-metric severity. The standard six carry severity; the +// flow-evidence metrics (concurrency, throttled, triggered) are evidence only (severity ok) +// and exist so a cause's metricIds resolve. +// --------------------------------------------------------------------------- + +export function buildMetrics(input: HealthInput): Metric[] { + const t = HEALTH_THRESHOLDS; + const ev = input.flowEvidence; + + const startLatency: Metric = { + id: "start_latency_p95", + value: input.startLatency.p95Ms, + unit: "ms", + aggregation: "p95", + normal: input.startLatency.normalP95Ms, + delta: delta(input.startLatency.p95Ms, input.startLatency.normalP95Ms), + series: { points: input.startLatency.series, kind: "measured" }, + severity: multiplierSeverity( + input.startLatency.p95Ms, + input.startLatency.normalP95Ms, + t.startLatency.warnMult, + t.startLatency.critMult, + t.startLatency.floor + ), + }; + + const pending: Metric = { + id: "pending", + value: input.pending.now, + unit: "count", + normal: input.pending.normal, + delta: delta(input.pending.now, input.pending.normal), + series: { + points: input.pending.series, + kind: input.pending.estimated ? "estimated" : "measured", + }, + severity: multiplierSeverity( + input.pending.now, + input.pending.normal, + t.pending.warnMult, + t.pending.critMult, + t.pending.floor + ), + }; + + const net = input.throughput.donePerMin - input.throughput.triggeredPerMin; + const throughput: Metric = { + id: "throughput", + value: net, + unit: "perMin", + aggregation: "rate", + breakdown: { done: input.throughput.donePerMin, triggered: input.throughput.triggeredPerMin }, + severity: net < 0 && isPendingIncreasing(input.pending.series) ? "warn" : "ok", + }; + + const failureSeverity: Severity = + input.failures.rate < t.failures.floorRate + ? "ok" + : multiplierSeverity( + input.failures.rate, + input.failures.normalRate, + t.failures.warnMult, + t.failures.critMult, + t.failures.floor + ); + const failures: Metric = { + id: "failures", + value: input.failures.rate, + unit: "ratio", + aggregation: "ratio", + normal: input.failures.normalRate, + delta: delta(input.failures.rate, input.failures.normalRate), + series: { points: input.failures.series, kind: "measured" }, + severity: failureSeverity, + }; + + const durP95: Metric = { + id: "dur_p95", + value: input.duration.p95Ms, + unit: "ms", + aggregation: "p95", + normal: input.duration.normalP95Ms, + delta: delta(input.duration.p95Ms, input.duration.normalP95Ms), + severity: multiplierSeverity( + input.duration.p95Ms, + input.duration.normalP95Ms, + t.duration.warnMult, + t.duration.critMult, + t.duration.floor + ), + }; + + const ageMs = input.liveness.telemetryAgeMs; + const livenessSeverity: Severity = + ageMs === null // no signal at all (brand-new/quiet env) — genuinely unknown, so NEUTRAL (ok): + ? "ok" // a fine-but-idle env must not surface as a yellow verdict, and it never + : // trust-guards. "lagging" (below) IS a real warn; "unknown" is not. + ageMs > t.liveness.staleMs + ? "crit" + : ageMs > t.liveness.freshMs + ? "warn" + : "ok"; + const liveness: Metric = { + id: "liveness", + // No signal -> value 0 is a placeholder, NOT a real "0ms fresh". `availability: "unknown"` + // says so, so a structured consumer never reads the 0 as freshness (the finding reason + // also carries "freshness_unknown"). A finite number keeps the JSON VM valid (no Infinity). + value: ageMs ?? 0, + availability: ageMs === null ? "unknown" : "measured", + unit: "ms", + severity: livenessSeverity, + }; + + // Flow-evidence metrics (severity ok — evidence, not a verdict). + const concurrency: Metric = { + id: "concurrency", + value: ev.runningSeries.length > 0 ? ev.runningSeries[ev.runningSeries.length - 1] : 0, + unit: "count", + breakdown: { limit: ev.envLimit }, + series: { points: ev.runningSeries, kind: "measured" }, + severity: "ok", + }; + const throttled: Metric = { + id: "throttled", + value: ev.throttledShare, + unit: "ratio", + severity: "ok", + }; + const triggered: Metric = { + id: "triggered", + value: input.throughput.triggeredPerMin, + unit: "perMin", + normal: input.throughput.normalTriggeredPerMin, + delta: delta(input.throughput.triggeredPerMin, input.throughput.normalTriggeredPerMin), + severity: "ok", + }; + + return [ + startLatency, + pending, + throughput, + failures, + durP95, + liveness, + concurrency, + throttled, + triggered, + ]; +} + +// --------------------------------------------------------------------------- +// Drain computation — shared by the flow policy (flow.ts) and the footer (health.ts). +// --------------------------------------------------------------------------- + +export function computeDrain(input: HealthInput): { drainMinutes: number; isDrainable: boolean } { + const donePerMin = input.throughput.donePerMin; + const drainMinutes = donePerMin === 0 ? Number.POSITIVE_INFINITY : input.pending.now / donePerMin; + return { + drainMinutes, + isDrainable: drainMinutes < HEALTH_THRESHOLDS.flowPolicy.drainCritMinutes, + }; +} diff --git a/apps/webapp/app/presenters/v3/reports/health/health-data.ts b/apps/webapp/app/presenters/v3/reports/health/health-data.ts new file mode 100644 index 00000000000..c1e9639d92d --- /dev/null +++ b/apps/webapp/app/presenters/v3/reports/health/health-data.ts @@ -0,0 +1,644 @@ +/** + * The `health` report's DATA layer — the ONLY place SQL / Redis IO lives. Loads a + * `HealthInput` (plain numbers) that the pure `interpret()` turns into a VM. + * + * Flow signal is sourced behind `FlowSource`: + * - QueueMetricsSource (preferred) — MEASURED queue depth + real scheduling delay + * (p95 wait) from `env_metrics`, which is env-level, so there is no per-queue split. + * - SnapshotFlowSource (fallback) — live Redis depth + an ESTIMATED backlog proxy + * and `runs.queued_duration`, used until the queue-metrics pipeline has populated + * `env_metrics` for this env. + * + * `flowSource` records which ran and drives `pending.estimated`, so the "informational + * only" caveat drops automatically on the measured path. Execution, liveness and + * throughput always come from `runs`. + */ + +import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { executeQuery, isQueryConcurrencyRejection } from "~/services/queryService.server"; +import { engine } from "~/v3/runEngine.server"; +import { HEALTH_THRESHOLDS, type HealthInput } from "./health"; + +/** §5.B — failures = user-code failures only (Expired/Canceled excluded from both sides). */ +const FAILURE_STATUSES = "'Failed','Crashed','System failure','Timed out'"; + +/** All terminal statuses — a run that reached any of these has left the queue (not backlog). */ +const FINISHED_STATUSES = + "'Completed','Canceled','Expired','Failed','Crashed','System failure','Timed out'"; + +/** p95 is index 3 of quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99) (1-based). */ +const WAIT_P95 = "quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]"; + +const BASELINE_PERIOD = "7d"; +const SPARKLINE_BUCKETS = 7; + +type Row = Record<string, unknown>; + +function num(value: unknown, fallback = 0): number { + const n = typeof value === "string" ? Number(value) : (value as number); + return Number.isFinite(n) ? n : fallback; +} + +function mean(xs: number[]): number { + return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length; +} + +/** Downsample a per-bucket series to ~N points so the sparkline width is stable (plan §5.C). */ +function resampleSeries(points: number[], target = SPARKLINE_BUCKETS): number[] { + if (points.length <= target) return points; + const out: number[] = []; + const stride = points.length / target; + for (let i = 0; i < target; i++) { + const start = Math.floor(i * stride); + const end = Math.floor((i + 1) * stride); + const slice = points.slice(start, Math.max(end, start + 1)); + out.push(mean(slice)); + } + return out; +} + +function failureRate(failures: number, completed: number): number { + const denom = failures + completed; + return denom === 0 ? 0 : failures / denom; +} + +/** + * Bug 1 fix — ClickHouse query concurrency cap. The query service rejects the 4th + * concurrent query per project (-> `runQuery` throws -> 500); this loader used to fire + * up to 4 at once. `mapWithConcurrency` runs `fn` over `items` with at most `limit` in + * flight, preserving order. We cap CH calls at 2 to leave headroom under the limit of 3 + * for other project traffic. (Redis calls don't count — kept outside this helper.) + */ +async function mapWithConcurrency<T, R>( + items: readonly T[], + limit: number, + fn: (item: T, index: number) => Promise<R> +): Promise<R[]> { + const results = new Array<R>(items.length); + let next = 0; + async function worker(): Promise<void> { + while (true) { + const index = next++; + if (index >= items.length) return; + results[index] = await fn(items[index], index); + } + } + const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker()); + await Promise.all(workers); + return results; +} + +/** Max ClickHouse queries this loader keeps in flight (limit of 3, leave headroom). */ +const CH_CONCURRENCY = 2; + +/** + * The per-project limit (3) is shared across ALL in-flight requests, not just this + * loader — so concurrent report requests can still transiently exceed it and get a + * rejection. It's retryable (a slot frees when another query finishes), so we back off + * and retry rather than surface a 500. Scheduling only — same query, same result. + */ +const CH_REJECTION_RETRIES = 6; +const CH_REJECTION_BACKOFF_MS = 60; // base for exponential "full jitter" backoff +const CH_REJECTION_BACKOFF_CAP_MS = 2000; // ceiling per attempt + +function isConcurrencyRejection(error: unknown): boolean { + // Prefer the query service's stable marker; fall back to message text for other shapes. + if (isQueryConcurrencyRejection(error)) return true; + const message = + error instanceof Error + ? error.message + : typeof error === "string" + ? error + : JSON.stringify(error ?? ""); + return /concurrency|too many|try again/i.test(message); +} + +/** rows + the actual (clip-aware) time window the query service resolved for this run. */ +type QueryResult = { rows: Row[]; timeRange: { from: Date; to: Date } }; + +/** Runs one (TRQL) report query. Injectable so tests can drive the loader with canned results. */ +export type HealthQueryRunner = ( + env: AuthenticatedEnvironment, + query: string, + period: string +) => Promise<QueryResult>; + +/** + * The loader's IO boundary (§7 Seam A): ClickHouse via the query service, Redis via the + * engine. Defaults wire the real singletons; overriding lets tests drive the loader's + * orchestration without booting the env-bound query-service client. + */ +export type HealthDeps = { + runQuery: HealthQueryRunner; + lengthOfEnvQueue: (env: AuthenticatedEnvironment) => Promise<number | undefined>; +}; + +/** + * The 7d baseline changes slowly, so cache it briefly (per env + query) to avoid recomputing a + * wide query on every request — the biggest lever on query pressure (#12). Only the default + * runner caches; injected test runners bypass this entirely, so test isolation is preserved. + */ +const BASELINE_CACHE_TTL_MS = 5 * 60_000; +const baselineCache = new Map<string, { expiresAt: number; result: QueryResult }>(); + +/** + * Store a baseline result, first sweeping expired entries so envs that stop requesting reports + * don't linger in memory forever. Writes only happen on a cache miss (~once per env per TTL), so + * a full sweep here is cheap and keeps the map bounded to recently-active environments. + */ +function cacheBaseline(key: string, result: QueryResult, now: number) { + for (const [k, v] of baselineCache) { + if (v.expiresAt <= now) baselineCache.delete(k); + } + baselineCache.set(key, { expiresAt: now + BASELINE_CACHE_TTL_MS, result }); +} + +async function executeReportQuery( + env: AuthenticatedEnvironment, + query: string, + period: string +): Promise<QueryResult> { + const cacheKey = period === BASELINE_PERIOD ? `${env.id}\u0000${query}` : null; + if (cacheKey) { + const hit = baselineCache.get(cacheKey); + if (hit && Date.now() < hit.expiresAt) return hit.result; + } + for (let attempt = 0; ; attempt++) { + const result = await executeQuery({ + name: "report-health", + query, + scope: "environment", + organizationId: env.organization.id, + projectId: env.project.id, + environmentId: env.id, + period, + history: { source: "API", skip: true }, + }); + if (result.success) { + const out = { rows: result.result.rows as Row[], timeRange: result.timeRange }; + if (cacheKey) cacheBaseline(cacheKey, out, Date.now()); + return out; + } + // Retry transient concurrency rejections; rethrow anything else (e.g. a bad query) so + // callers/tryQuery can handle it. Exponential "full jitter" backoff — delay picked uniformly + // from [0, min(cap, base·2^attempt)) — so concurrent report requests don't wake in lockstep + // and re-collide on the same 3 query slots. + if (attempt < CH_REJECTION_RETRIES && isConcurrencyRejection(result.error)) { + const window = Math.min(CH_REJECTION_BACKOFF_CAP_MS, CH_REJECTION_BACKOFF_MS * 2 ** attempt); + await new Promise((resolve) => setTimeout(resolve, Math.floor(Math.random() * window))); + continue; + } + throw result.error; + } +} + +// --------------------------------------------------------------------------- +// runs queries (execution + liveness + throughput; also feed the snapshot fallback). +// executeQuery injects tenant isolation + the time window, so we never write WHERE. +// --------------------------------------------------------------------------- + +function runsScalarQuery(): string { + return `SELECT + quantile(0.95)(queued_duration) AS start_latency_p95, + quantile(0.95)(execution_duration) AS dur_p95, + countIf(status IN (${FAILURE_STATUSES})) AS failures, + countIf(status = 'Completed') AS completed, + count() AS triggered, + max(triggered_at) AS last_activity +FROM runs`; +} + +function runsSeriesQuery(): string { + return `SELECT + timeBucket() AS t, + quantile(0.95)(queued_duration) AS start_latency_p95, + countIf(status IN (${FAILURE_STATUSES})) AS failures, + countIf(status = 'Completed') AS completed, + countIf(status IN (${FINISHED_STATUSES})) AS finished, + count() AS triggered +FROM runs +GROUP BY t +ORDER BY t`; +} + +// --------------------------------------------------------------------------- +// env_metrics queries (measured queue depth + scheduling delay). +// --------------------------------------------------------------------------- + +function envSeriesQuery(): string { + return `SELECT + timeBucket() AS t, + max(max_env_queued) AS queued, + max(max_env_running) AS running, + sum(throttled_count) AS throttled, + ${WAIT_P95} AS wait_p95 +FROM env_metrics +GROUP BY t +ORDER BY t`; +} + +function envScalarQuery(): string { + return `SELECT + ${WAIT_P95} AS wait_p95, + avg(max_env_queued) AS avg_queued, + max(max_env_limit) AS env_limit, + max(bucket_start) AS last_bucket +FROM env_metrics`; +} + +/** + * Worst queue by share of CURRENT pending. `argMax(max_queued, bucket_start)` = each queue's + * depth in its latest bucket, so the shares sum to a real point-in-time backlog — not a sum of + * per-queue peaks from different moments (which isn't "% of pending" at any instant). Best-effort. + */ +function queueWorstQuery(): string { + return `SELECT + queue AS name, + argMax(max_queued, bucket_start) AS latest_queued +FROM queue_metrics +GROUP BY queue +ORDER BY latest_queued DESC +LIMIT 20`; +} + +/** + * Runs dead-lettered across the window, summed over queues. `dlq_delta` is per-queue + * cumulative-counter state, so it must be merged per queue then summed (never merged + * across queues). Best-effort — absent columns just yield no rows. + */ +function dlqTotalQuery(): string { + return `SELECT sum(dlq) AS dlq_total +FROM ( + SELECT deltaSumTimestampMerge(dlq_delta) AS dlq + FROM queue_metrics + GROUP BY queue +)`; +} + +/** Top failing task (lazy — only when execution degrades). Best-effort. */ +function failureBreakdownQuery(): string { + return `SELECT + task_identifier AS task, + countIf(status IN (${FAILURE_STATUSES})) AS fails +FROM runs +GROUP BY task +ORDER BY fails DESC +LIMIT 10`; +} + +/** Default IO wiring — the real query-service runner + the engine's env-queue length. */ +const defaultHealthDeps: HealthDeps = { + runQuery: executeReportQuery, + lengthOfEnvQueue: (env) => engine.lengthOfEnvQueue(env), +}; + +/** Run a query that may reference not-yet-available columns; never break the report. */ +async function tryQuery( + deps: HealthDeps, + env: AuthenticatedEnvironment, + query: string, + period: string +): Promise<Row[]> { + try { + return (await deps.runQuery(env, query, period)).rows; + } catch { + return []; + } +} + +// --------------------------------------------------------------------------- +// FlowSource seam (§7 Seam A). +// --------------------------------------------------------------------------- + +export type FlowData = { + flowSource: HealthInput["flowSource"]; + pending: { now: number; normal?: number; series: number[]; estimated: boolean }; + startLatency: { p95Ms: number; normalP95Ms: number; series: number[] }; + evidence: HealthInput["flowEvidence"]; + /** + * Epoch ms of the freshest telemetry the source saw (latest env_metrics bucket and/or latest + * run) — how the report tells "data current" from "pipeline stale", independent of traffic. + * null when no signal exists at all (brand-new/empty env) -> liveness "unknown", not stale. + */ + telemetryLastTs: number | null; +}; + +const EMPTY_EVIDENCE: HealthInput["flowEvidence"] = { + runningSeries: [], + envLimit: 0, + throttledShare: 0, + worstQueue: null, + dlqDelta: null, // snapshot path: dead-letter volume is unmeasured +}; + +/** The runs results loadHealthInput already fetched, so the snapshot fallback needn't re-query. */ +type RunsContext = { liveScalar: Row; liveSeries: Row[]; baselineScalar: Row }; + +export interface FlowSource { + loadFlow( + env: AuthenticatedEnvironment, + period: string, + ctx: RunsContext, + deps: HealthDeps + ): Promise<FlowData | null>; +} + +/** + * Preferred source: measured queue depth + scheduling-delay p95 from `env_metrics`. + * Returns null when the pipeline hasn't populated the table yet, so the caller can fall + * back to the snapshot. + */ +export const QueueMetricsSource: FlowSource = { + async loadFlow(env, period, ctx, deps) { + try { + // Redis depth isn't a CH query, so it runs alongside (doesn't count toward the cap). + // Guard the rejection: if the CH queries throw first we jump to catch without awaiting + // this, and an unhandled Redis rejection would crash the process. + const pendingNowPromise = deps.lengthOfEnvQueue(env).catch(() => undefined); + + // Bug 1 fix — route all CH queries through the concurrency cap (max 2 in flight). + // Every task returns Row[] (scalars indexed after) so mapWithConcurrency infers a + // single element type — a mixed Row[]/Row union trips its generic inference. + const [seriesRows, liveScalarRows, baselineScalarRows, worstQueueRows, dlqResultRows] = + await mapWithConcurrency( + [ + () => deps.runQuery(env, envSeriesQuery(), period).then((r) => r.rows), + () => deps.runQuery(env, envScalarQuery(), period).then((r) => r.rows), + () => deps.runQuery(env, envScalarQuery(), BASELINE_PERIOD).then((r) => r.rows), + () => tryQuery(deps, env, queueWorstQuery(), period), + () => tryQuery(deps, env, dlqTotalQuery(), period), + ], + CH_CONCURRENCY, + (task) => task() + ); + const liveScalarRow = liveScalarRows[0] ?? {}; + const baselineScalarRow = baselineScalarRows[0] ?? {}; + + const pendingNow = await pendingNowPromise; + + if (seriesRows.length === 0) { + return null; // no measured data yet -> snapshot fallback + } + + // Telemetry freshness = the freshest of the latest env_metrics bucket (a heartbeat + // independent of traffic) and the latest run recorded. + const telemetryLastTs = freshestTs(liveScalarRow.last_bucket, ctx.liveScalar.last_activity); + + return buildQueueMetricsFlow( + seriesRows, + liveScalarRow, + baselineScalarRow, + worstQueueRows, + dlqResultRows, + pendingNow, + telemetryLastTs + ); + } catch { + // Bug 2 fix — if `env_metrics` isn't available the queries throw; return null so + // loadHealthInput falls back to the snapshot instead of propagating a 500. + return null; + } + }, +}; + +function buildQueueMetricsFlow( + series: Row[], + liveScalar: Row, + baselineScalar: Row, + worstRows: Row[], + dlqRows: Row[], + pendingNow: number | undefined, + telemetryLastTs: number | null +): FlowData { + // Dead-letter volume (0 = measured none; no rows -> unmeasured -> null). + const dlqDelta = dlqRows.length > 0 ? Math.round(num(dlqRows[0].dlq_total)) : null; + + // Throttled share = fraction of buckets with any queue-level throttling. + const throttledShare = + series.length > 0 ? series.filter((r) => num(r.throttled) > 0).length / series.length : 0; + + // Worst queue = top queue's share of current pending (latest-bucket depths, so shares + // sum to a real point-in-time backlog). + let worstQueue: HealthInput["flowEvidence"]["worstQueue"] = null; + if (worstRows.length > 0) { + const depths = worstRows.map((r) => num(r.latest_queued)); + const total = depths.reduce((a, b) => a + b, 0); + if (total > 0) { + worstQueue = { name: String(worstRows[0].name ?? "unknown"), share: depths[0] / total }; + } + } + + // Prefer live Redis depth; if it's unavailable fall back to the latest MEASURED queued from + // env_metrics (still a real number) rather than a misleading confident zero (#7). + const lastMeasuredQueued = num(series[series.length - 1]?.queued); + + return { + flowSource: "queue_metrics_v1", + pending: { + now: pendingNow ?? lastMeasuredQueued, + normal: Math.round(num(baselineScalar.avg_queued)), + series: resampleSeries(series.map((r) => num(r.queued))), + estimated: false, // measured + }, + startLatency: { + p95Ms: num(liveScalar.wait_p95), + normalP95Ms: num(baselineScalar.wait_p95), + series: resampleSeries(series.map((r) => num(r.wait_p95))), + }, + evidence: { + // native resolution — cause discriminators read shares off this series. + runningSeries: series.map((r) => num(r.running)), + envLimit: num(liveScalar.env_limit), + throttledShare, + worstQueue, + dlqDelta, + }, + telemetryLastTs, + }; +} + +/** + * Fallback: live Redis depth (accurate "now") + an estimated backlog proxy from `runs` + * (cumulative triggered - finished) and `runs.queued_duration` for latency. The proxy is a + * shape-only TREND (`estimated: true`): it starts at 0 within the window and can't see + * backlog that predates it. + */ +export const SnapshotFlowSource: FlowSource = { + async loadFlow(env, _period, ctx, deps) { + // Guard Redis: this is the last-resort source, so a failure must not break the report. + const pendingNow = (await deps.lengthOfEnvQueue(env).catch(() => undefined)) ?? 0; + + // Subtract ALL terminal runs, not just Completed — else failed/expired/canceled runs + // linger in the proxy as phantom backlog forever. + let backlog = 0; + const proxy = ctx.liveSeries.map((r) => { + backlog = Math.max(0, backlog + num(r.triggered) - num(r.finished)); + return backlog; + }); + const series = resampleSeries(proxy); + + return { + flowSource: "snapshot+runs", + pending: { + now: pendingNow, + // No 7d pending baseline on this path — omit `normal` rather than pass off a + // live-window proxy average as "7d normal" (#8). Severity falls back to an absolute floor. + normal: undefined, + series, + estimated: true, + }, + startLatency: { + p95Ms: num(ctx.liveScalar.start_latency_p95), + normalP95Ms: num(ctx.baselineScalar.start_latency_p95), + series: resampleSeries(ctx.liveSeries.map((r) => num(r.start_latency_p95))), + }, + // No cause-tree evidence; interpret falls back to v1 symptoms. + evidence: EMPTY_EVIDENCE, + // No env_metrics heartbeat here — the only freshness signal is the latest run recorded + // (null when the env has no runs at all -> liveness "unknown", not stale). + telemetryLastTs: freshestTs(ctx.liveScalar.last_activity), + }; + }, +}; + +// --------------------------------------------------------------------------- +// loadHealthInput. +// --------------------------------------------------------------------------- + +export async function loadHealthInput( + env: AuthenticatedEnvironment, + period: string, + now: Date = new Date(), + deps: HealthDeps = defaultHealthDeps +): Promise<HealthInput> { + // Bug 1 fix — route the runs-phase CH queries through the concurrency cap (max 2) + // instead of firing all 3 at once, so we never exceed the per-project limit. + const [liveScalarRes, liveSeriesRes, baselineScalarRes] = await mapWithConcurrency( + [ + () => deps.runQuery(env, runsScalarQuery(), period), + () => deps.runQuery(env, runsSeriesQuery(), period), + () => deps.runQuery(env, runsScalarQuery(), BASELINE_PERIOD), + ], + CH_CONCURRENCY, + (task) => task() + ); + const ctx: RunsContext = { + liveScalar: liveScalarRes.rows[0] ?? {}, + liveSeries: liveSeriesRes.rows, + baselineScalar: baselineScalarRes.rows[0] ?? {}, + }; + + // Window lengths come from the query service's resolved (clip-aware) range, not a + // re-parse of the period — so `maxQueryPeriod` clipping can't skew per-minute rates + // or annotation math. periodToMinutes is a fallback for a degenerate range. + const windowMinutes = timeRangeMinutes(liveSeriesRes.timeRange) || periodToMinutes(period); + const baselineMinutes = + timeRangeMinutes(baselineScalarRes.timeRange) || periodToMinutes(BASELINE_PERIOD); + + // Prefer measured queue metrics; fall back to the runs snapshot when unavailable. + const flow = + (await QueueMetricsSource.loadFlow(env, period, ctx, deps)) ?? + (await SnapshotFlowSource.loadFlow(env, period, ctx, deps))!; + + const failuresSeries = resampleSeries( + ctx.liveSeries.map((r) => failureRate(num(r.failures), num(r.completed))) + ); + + const triggered = num(ctx.liveScalar.triggered); + const completed = num(ctx.liveScalar.completed); + const donePerMin = windowMinutes === 0 ? 0 : completed / windowMinutes; + const triggeredPerMin = windowMinutes === 0 ? 0 : triggered / windowMinutes; + const normalTriggeredPerMin = + baselineMinutes === 0 ? 0 : num(ctx.baselineScalar.triggered) / baselineMinutes; + + const rate = failureRate(num(ctx.liveScalar.failures), num(ctx.liveScalar.completed)); + const normalRate = failureRate( + num(ctx.baselineScalar.failures), + num(ctx.baselineScalar.completed) + ); + + // Lazy failure attribution — only when execution is actually degraded. + // A fresh failure pattern from a clean 0% baseline is exactly when attribution matters most, + // so a zero baseline (can't form a ratio) counts as degraded once past the floor. + const failureDegraded = + rate >= HEALTH_THRESHOLDS.failures.floorRate && + (normalRate === 0 || rate / normalRate >= HEALTH_THRESHOLDS.failures.warnMult); + const failureBreakdown = failureDegraded + ? await loadFailureBreakdown(deps, env, period, num(ctx.liveScalar.failures)) + : undefined; + + // Liveness = telemetry freshness (how recent is the newest data), NOT "recent completions": + // a quiet env with a fresh pipeline is fresh; a dead pipeline is stale. null -> unknown. + const telemetryAgeMs = + flow.telemetryLastTs === null ? null : Math.max(0, now.getTime() - flow.telemetryLastTs); + + return { + scope: env.slug ?? "environment", + period: humanPeriod(period), + baselineLabel: `vs your ${BASELINE_PERIOD} normal`, + generatedAt: now.toISOString(), + windowMinutes, + flowSource: flow.flowSource, + pending: flow.pending, + startLatency: flow.startLatency, + throughput: { donePerMin, triggeredPerMin, normalTriggeredPerMin }, + failures: { rate, normalRate, series: failuresSeries }, + duration: { + p95Ms: num(ctx.liveScalar.dur_p95), + normalP95Ms: num(ctx.baselineScalar.dur_p95), + }, + liveness: { telemetryAgeMs }, + flowEvidence: flow.evidence, + failureBreakdown, + }; +} + +async function loadFailureBreakdown( + deps: HealthDeps, + env: AuthenticatedEnvironment, + period: string, + totalFails: number +): Promise<HealthInput["failureBreakdown"]> { + if (totalFails <= 0) return undefined; + const rows = await tryQuery(deps, env, failureBreakdownQuery(), period); + if (rows.length === 0) return undefined; + const top = rows[0]; + return { task: String(top.task ?? "unknown"), share: num(top.fails) / totalFails }; +} + +// --------------------------------------------------------------------------- +// Small local helpers. +// --------------------------------------------------------------------------- + +function parseTimestamp(value: unknown): number | null { + if (value === null || value === undefined || value === "") return null; + // ClickHouse returns a "1970-01-01 00:00:00" sentinel for max() over no rows. + const ts = Date.parse(String(value).replace(" ", "T") + (String(value).includes("Z") ? "" : "Z")); + if (!Number.isFinite(ts) || ts <= 0) return null; + return ts; +} + +/** The most recent of several timestamp cells (epoch ms), or null when none parse. */ +function freshestTs(...values: unknown[]): number | null { + const times = values.map(parseTimestamp).filter((t): t is number => t !== null); + return times.length === 0 ? null : Math.max(...times); +} + +/** Minutes spanned by a resolved query time range (0 if degenerate, so callers can fall back). */ +function timeRangeMinutes(range: { from: Date; to: Date }): number { + const ms = range.to.getTime() - range.from.getTime(); + return ms > 0 ? Math.round(ms / 60_000) : 0; +} + +function periodToMinutes(period: string): number { + const match = /^(\d+)\s*([smhdw])$/.exec(period.trim()); + if (!match) return 60; + const value = Number(match[1]); + const unit = match[2]; + const minutes: Record<string, number> = { s: 1 / 60, m: 1, h: 60, d: 1440, w: 10080 }; + return value * (minutes[unit] ?? 60); +} + +function humanPeriod(period: string): string { + return `last ${period}`; +} diff --git a/apps/webapp/app/presenters/v3/reports/health/health-messages.ts b/apps/webapp/app/presenters/v3/reports/health/health-messages.ts new file mode 100644 index 00000000000..888b2b54b73 --- /dev/null +++ b/apps/webapp/app/presenters/v3/reports/health/health-messages.ts @@ -0,0 +1,170 @@ +/** + * The `health` report's PROSE — the single place its codes resolve to strings. Registered + * under the "health" title so the generic renderer can look it up by `vm.title` without ever + * importing health vocabulary. A future report (Cost, Regression) ships its own catalog the + * same way; `report-messages.ts` stays report-agnostic infrastructure. + * + * Some strings carry {tokens} (e.g. {age}, {rate}) the renderer fills from the finding's + * metrics / evidence — meaning lives here, numbers stay facts. + */ + +import { registerReportMessages, type ReportMessages } from "../report-messages"; +import { type ReasonCode, type Severity } from "../report-view-model"; + +/** Metric id -> expanded display label. */ +const METRIC_LABELS: Record<string, string> = { + start_latency_p95: "start latency", + pending: "pending", + throughput: "throughput", + failures: "failures", + dur_p95: "p95 dur", + liveness: "liveness", + concurrency: "concurrency", + throttled: "throttled", + triggered: "triggered", +}; + +/** + * Finding headline — keyed by `${findingType}/${reason}`. Degraded = the cause, + * healthy = reassurance. `@expanded` variants show a healthy finding expanded + * (e.g. execution while flow is degraded). + */ +const FINDING_REASONS: Record<string, string> = { + // flow — causes + "flow/env_limit_saturation": "at your env concurrency limit", + "flow/dequeue_stall": "capacity is free but nothing is dequeuing", + "flow/queue_limit_throttling": "a queue is throttling at its own limit", + "flow/trigger_spike": "a trigger spike is backing up the queue", + "flow/trigger_surge": "a surge of new triggers is backing up the queue", + // flow — fallback symptoms (v1) + "flow/start_latency": "runs are slow to start", + "flow/backlog": "backlog is growing", + "flow/throughput_lag": "completion is falling behind triggers", + "flow/degraded": "flow is degraded", + // flow — healthy (collapsed) + "flow/healthy": "starting normally", + // execution + "execution/failures_up": "runs are failing more than usual", + "execution/slow_runs": "runs are slower than usual", + "execution/degraded": "execution is degraded", + "execution/unknown": "execution can't be assessed — the telemetry is stale", + "flow/unknown": "flow can't be assessed — the telemetry is stale", + "execution/healthy": "completing normally", // collapsed + "execution/healthy@expanded": "the runs that DO start are fine", + // liveness = telemetry freshness ({age} filled by the renderer) + "liveness/fresh": "fresh — telemetry current, updated {age} ago", + "liveness/lagging": "lagging — telemetry last updated {age} ago", + "liveness/stale": "stale — no telemetry in {age}", + "liveness/freshness_unknown": "freshness unknown — no telemetry signal to check", +}; + +/** The "read:" causal-chain line — keyed by code. */ +const READS: Record<string, string> = { + // cause chains + saturation_chain: "limit saturated → incoming work exceeds capacity → backlog grows", + capacity_free_not_dequeuing: "capacity is free but work isn't being picked up — on our side", + queue_throttle_chain: "queue at its limit → its runs wait → backlog grows", + spike_chain: "triggers jumped {mult}× → queue fills faster than it drains", + surge_chain: "new triggers arriving with no prior baseline → queue fills faster than it drains", + // fallback symptoms (v1) + starting_normally: "runs are starting on time", + lag_while_triggering_normal: "triggering normally, but starts lag → work is backing up", + lag_and_failures: "runs are lagging AND failing — check the code path", + degraded_generic: "flow is degraded", + not_a_code_problem: "NOT a code problem", + runs_are_fine: "runs are completing normally", + failures_elevated: "failures are elevated — check the code path", + data_stale: "data is stale — the verdict cannot be trusted", +}; + +/** Exclusion (ruled-out cause) — rendered under `read:`. {tokens} filled from evidence. */ +const EXCLUSIONS: Record<string, string> = { + not_env_limit: "env concurrency limit is not the bottleneck", + not_your_code: "not your code — failures and durations normal", + not_your_config: "not your config — limits aren't the bottleneck", +}; + +/** Observation (supporting fact, not a ruled-out cause) — rendered under `read:` after exclusions. */ +const OBSERVATIONS: Record<string, string> = { + not_workers_platform: "runs are completing at ~{rate}/min", + execution_healthy: "runs that start are completing normally", + nothing_dead_lettered: "nothing dead-lettered", +}; + +/** Metric annotation shown on a cause line INSTEAD of "(normal ~x)". {tokens} filled by the renderer. */ +const ANNOTATIONS: Record<string, string> = { + pinned_minutes: "pinned {value} of last {window} min", + idle_share: "idle — {value} running of {limit}", + throttled_minutes: "throttled {value} of last {window} min", + spike_mult: "{value}× the normal rate", + surge_rate: "{value}/min, no prior baseline", +}; + +/** Headline statement — keyed by `${findingType}/${severity}`. */ +const STATEMENTS: Record<string, string> = { + "flow/ok": "Flow healthy", + "flow/warn": "Flow slowing", + "flow/crit": "Flow stalled", + "execution/ok": "Execution healthy", + "execution/warn": "Execution degraded", + "execution/crit": "Execution failing", + "liveness/ok": "data fresh", + "liveness/warn": "data lagging", + "liveness/crit": "data stale", +}; + +/** Recommendation / footer codes -> calm, jargon-free action text. */ +const ACTIONS: Record<string, string> = { + // Review = open concrete data · Check = system state · Raise = a settings change + review_start_latency: "Review start latency", + review_failing_tasks: "Review failing tasks", + review_slow_runs: "Review slow runs", + review_trigger_source: "Review what's triggering the spike", + check_queue_health: "Check queue health", + check_worker_availability: "Check worker availability", + check_control_plane: "Check control plane", + check_platform_status: "Check status.trigger.dev — no action needed on yours", + raise_env_limit: "Raise the env concurrency limit", + raise_queue_limit: "Raise the queue's concurrency limit", + do_nothing_drains: "or do nothing — backlog drains in ~{value} min once triggers ease", + region_failover: "region move? ask your agent — depends on your failover setup", + nothing_to_do: "nothing to do", +}; + +function findingReason( + findingType: string, + reason: ReasonCode, + opts?: { expanded?: boolean } +): string { + if (opts?.expanded) { + const expanded = FINDING_REASONS[`${findingType}/${reason}@expanded`]; + if (expanded) return expanded; + } + return FINDING_REASONS[`${findingType}/${reason}`] ?? reason; +} + +function statementMessage(findingType: string, severity: Severity, reason?: ReasonCode): string { + // Stale telemetry makes a CH-derived verdict untrustworthy — say so, don't show a severity. + if (reason === "unknown") { + const label = findingType.charAt(0).toUpperCase() + findingType.slice(1); + return `${label} unknown — data stale`; + } + // No freshness signal is NOT "data lagging" (a real severity) — it's genuinely unknown. + if (findingType === "liveness" && reason === "freshness_unknown") { + return "data freshness unknown"; + } + return STATEMENTS[`${findingType}/${severity}`] ?? `${findingType} ${severity}`; +} + +export const healthMessages: ReportMessages = { + metricLabel: (id) => METRIC_LABELS[id] ?? id, + findingReason, + readMessage: (code) => READS[code] ?? code, + exclusionMessage: (code) => EXCLUSIONS[code] ?? code, + observationMessage: (code) => OBSERVATIONS[code] ?? code, + annotationMessage: (code) => ANNOTATIONS[code] ?? code, + statementMessage, + actionMessage: (code) => ACTIONS[code] ?? code, +}; + +registerReportMessages("health", healthMessages); diff --git a/apps/webapp/app/presenters/v3/reports/health/health.ts b/apps/webapp/app/presenters/v3/reports/health/health.ts new file mode 100644 index 00000000000..08beae3894c --- /dev/null +++ b/apps/webapp/app/presenters/v3/reports/health/health.ts @@ -0,0 +1,268 @@ +/** + * The `health` report — ORCHESTRATOR + public entry. Two layers: + * `assessHealth()` data -> HealthAssessment (all health reasoning) + * `toReportViewModel()` HealthAssessment -> generic ReportViewModel (pure packaging) + * `interpret()` is the thin composition of the two. + * + * The reasoning is split into three independent analyzers, each returning a Finding: + * flow.ts · execution.ts · liveness.ts (foundation in health-core.ts) + * This module only wires them: build metrics -> run the three -> flow policy -> stale guard + * -> reads -> summary/footer. PURE — no IO/clock/LLM/formatting. + */ + +import { + isOk, + maxSeverity, + type Finding, + type FooterEntry, + type Metric, + type ReportViewModel, + type Severity, + type SummaryStatement, +} from "../report-view-model"; +import { buildMetrics, computeDrain, HEALTH_THRESHOLDS, type HealthInput } from "./health-core"; +import { buildExecutionRead, interpretExecution } from "./execution"; +import { applyFlowPolicy, buildFlowRead, interpretFlow } from "./flow"; +import { interpretLiveness } from "./liveness"; +// Registers the "health" message catalog (side effect) so the renderer resolves this report's +// codes. Kept here — the health report's entry module — so loading it always registers its prose. +import "./health-messages"; + +// Re-exported so the data layer + tests keep a single import path (`./health`). +export { HEALTH_THRESHOLDS, isPendingIncreasing, type HealthInput } from "./health-core"; + +// --------------------------------------------------------------------------- +// Stale-telemetry trust guard. When telemetry is genuinely stale, both flow and execution are +// CH-derived and untrustworthy: mark them unknown and strip everything that would advise action +// off stale data (recommendation, attribution, exclusions, observations, hedge, anomaly window). +// --------------------------------------------------------------------------- + +function applyStaleGuard( + flow: Finding, + execution: Finding, + telemetryStale: boolean +): { flow: Finding; execution: Finding; treatedCrit: boolean } { + if (!telemetryStale) return { flow, execution, treatedCrit: false }; + // Force crit so severity is consistent across summary / section glyph / JSON, and strip the + // ACTIONABLE causal fields so no surface advises off stale data. Raw metrics/evidence stay in + // the VM for diagnostics, flagged informational-only by `facts.trustworthy: false`. + const untrust = (f: Finding): Finding => ({ + ...f, + severity: "crit", + reason: "unknown", + recommendation: undefined, + attribution: undefined, + exclusions: undefined, + observations: undefined, + hedge: undefined, + anomalyWindow: undefined, + }); + return { flow: untrust(flow), execution: untrust(execution), treatedCrit: true }; +} + +// --------------------------------------------------------------------------- +// Summary. +// --------------------------------------------------------------------------- + +function aggregateSummary( + flow: Finding, + execution: Finding, + liveness: Finding, + executionTreatedCrit: boolean +): { severity: Severity; statements: SummaryStatement[] } { + const executionSummarySeverity: Severity = executionTreatedCrit ? "crit" : execution.severity; + const severity = maxSeverity(flow.severity, executionSummarySeverity, liveness.severity); + const statements: SummaryStatement[] = [ + { + findingType: "flow", + severity: flow.severity, + reason: flow.reason === "unknown" ? "unknown" : undefined, + }, + { + findingType: "execution", + severity: executionSummarySeverity, + reason: execution.reason === "unknown" ? "unknown" : undefined, + }, + { + findingType: "liveness", + severity: liveness.severity, + reason: liveness.reason === "freshness_unknown" ? "freshness_unknown" : undefined, + }, + ]; + return { severity, statements }; +} + +// --------------------------------------------------------------------------- +// Footer (dominant action + do-nothing option) + links. +// --------------------------------------------------------------------------- + +const SEV_RANK: Record<Severity, number> = { ok: 0, warn: 1, crit: 2 }; + +function dominantFinding(findings: Finding[]): Finding | undefined { + let best: Finding | undefined; + for (const f of findings) { + if (!f.recommendation) continue; + if (!best || SEV_RANK[f.severity] > SEV_RANK[best.severity]) best = f; + } + return best; +} + +function buildFooter( + findings: Finding[], + drain: { drainMinutes: number; isDrainable: boolean }, + telemetryStale: boolean +): FooterEntry[] { + // Stale telemetry: the only trustworthy action is to check the pipeline itself. + if (telemetryStale) { + const liveness = findings.find((f) => f.type === "liveness"); + return liveness?.recommendation + ? [{ code: liveness.recommendation.code, link: liveness.recommendation.link }] + : [{ code: "nothing_to_do" }]; + } + + const dominant = dominantFinding(findings); + if (!dominant?.recommendation) return [{ code: "nothing_to_do" }]; + + const footer: FooterEntry[] = [ + { code: dominant.recommendation.code, link: dominant.recommendation.link }, + ]; + + // Second entry: do-nothing when the backlog drains, or the region-move hedge for a + // dequeue stall (the one place it stays plausible). + if (dominant.type === "flow") { + if (drain.isDrainable && Number.isFinite(drain.drainMinutes)) { + footer.push({ code: "do_nothing_drains", value: Math.round(drain.drainMinutes * 10) / 10 }); + } else if (dominant.reason === "dequeue_stall") { + footer.push({ code: "region_failover" }); + } + } + return footer; +} + +function collectLinks(findings: Finding[]): ReportViewModel["links"] { + const keys = new Set<string>(); + for (const finding of findings) { + if (finding.recommendation?.link) keys.add(finding.recommendation.link); + if (finding.hedge?.link) keys.add(finding.hedge.link); + } + return [...keys].map((key) => ({ key, label: key, url: "" })); +} + +// --------------------------------------------------------------------------- +// Domain layer: HealthAssessment — the health verdict (flow / execution / liveness findings, +// their causes + recommendations, and the derived state). All health semantics live here; it +// knows nothing about how a report is presented. `toReportViewModel` maps it into the generic, +// report-agnostic ReportViewModel — so the VM stays reusable for future reports (each has its +// own <domain>Assessment + mapper; the renderer/VM primitives are shared). +// --------------------------------------------------------------------------- + +export type HealthAssessment = { + /** header, carried through from the input. */ + scope: string; + period: string; + baselineLabel: string; + generatedAt: string; + windowMinutes: number; + /** finalized findings (post-policy, post-stale-guard, with reads built). */ + flow: Finding; + execution: Finding; + liveness: Finding; + metrics: Metric[]; + /** derived domain state the presentation layer needs (footer / summary / trust). */ + drain: { drainMinutes: number; isDrainable: boolean }; + telemetryStale: boolean; + executionTreatedCrit: boolean; + /** structured payload for agents (already carries the trust marker). */ + facts: Record<string, unknown>; +}; + +/** Data -> domain verdict. Pure: no IO/clock/LLM/formatting — just health reasoning. */ +export function assessHealth(input: HealthInput): HealthAssessment { + const metrics = buildMetrics(input); + const drain = computeDrain(input); + + let flow = interpretFlow(metrics, input); + const executionRaw = interpretExecution(metrics, input); + const liveness = interpretLiveness(metrics, input); + + // Telemetry freshness as an explicit state so "unknown" (no signal) is never conflated with + // "lagging" (a real severity). Only GENUINE staleness trust-guards the CH-derived verdicts. + const ageMs = input.liveness.telemetryAgeMs; + const telemetryStale = ageMs !== null && ageMs > HEALTH_THRESHOLDS.liveness.staleMs; + + flow = applyFlowPolicy(flow, executionRaw, drain.isDrainable, telemetryStale); + + // Stale telemetry: flow AND execution are untrustworthy -> mark both unknown and strip their + // actions/attribution/exclusions so nothing advises off stale data. + const guarded = applyStaleGuard(flow, executionRaw, telemetryStale); + flow = guarded.flow; + const execution = guarded.execution; + + // interpretFlow mutates the shared metrics array (e.g. sets concurrency.annotation "pinned 40 + // of last 60 min") BEFORE the guard runs. The renderers hide it for an unknown finding, but + // format=json would still leak that stale-derived narrative — so strip metric annotations too + // (the twin of the stripped anomaly window). Raw values stay, flagged by facts.trustworthy. + if (telemetryStale) { + for (const m of metrics) m.annotation = undefined; + } + + // Reads built last, from the finalized findings. + flow.read = buildFlowRead(flow, isOk(execution.severity), !telemetryStale); + execution.read = buildExecutionRead(execution, flow); + + return { + scope: input.scope, + period: input.period, + baselineLabel: input.baselineLabel, + generatedAt: input.generatedAt, + windowMinutes: input.windowMinutes, + flow, + execution, + liveness, + metrics, + drain, + telemetryStale, + executionTreatedCrit: guarded.treatedCrit, + facts: { + // Trust marker for structured consumers. The metrics/evidence stay (useful for pipeline + // diagnostics), but when telemetry is stale they're informational-only: an agent must not + // act on them (e.g. raise concurrency off a stale backlog). The human renderer is already + // guarded via the "unknown" finding; this is the same guarantee for JSON. + trustworthy: !telemetryStale, + staleReason: telemetryStale ? "telemetry_stale" : undefined, + flowSource: input.flowSource, + pendingEstimated: input.pending.estimated, + throughput: input.throughput, + flowEvidence: input.flowEvidence, + }, + }; +} + +// --------------------------------------------------------------------------- +// Presentation mapping: HealthAssessment -> generic ReportViewModel. Pure packaging only — +// no health reasoning here (summary/footer/links are derived from the findings). +// --------------------------------------------------------------------------- + +function toReportViewModel(a: HealthAssessment): ReportViewModel { + const findings = [a.flow, a.execution, a.liveness]; + return { + title: "health", + scope: a.scope, + period: a.period, + baselineLabel: a.baselineLabel, + generatedAt: a.generatedAt, + windowMinutes: a.windowMinutes, + summary: aggregateSummary(a.flow, a.execution, a.liveness, a.executionTreatedCrit), + findings, + metrics: a.metrics, + facts: a.facts, + links: collectLinks(findings), + // Stale telemetry -> footer points at the control plane, not a CH-derived action. + footer: buildFooter(findings, a.drain, a.telemetryStale), + }; +} + +/** Public entry: data -> generic report. Thin composition of the domain + presentation layers. */ +export function interpret(input: HealthInput): ReportViewModel { + return toReportViewModel(assessHealth(input)); +} diff --git a/apps/webapp/app/presenters/v3/reports/health/liveness.ts b/apps/webapp/app/presenters/v3/reports/health/liveness.ts new file mode 100644 index 00000000000..4c3e313fdaa --- /dev/null +++ b/apps/webapp/app/presenters/v3/reports/health/liveness.ts @@ -0,0 +1,27 @@ +/** + * LIVENESS analyzer: telemetry FRESHNESS (age of the freshest signal), NOT "last completion". + * A null age is genuinely unknown (neutral), never a warning; only real staleness is crit. + */ + +import { type Finding, type Metric } from "../report-view-model"; +import { metricById, type HealthInput } from "./health-core"; + +export function interpretLiveness(metrics: Metric[], input: HealthInput): Finding { + const liveness = metricById(metrics, "liveness"); + const unknown = input.liveness.telemetryAgeMs === null; + const reason = unknown + ? "freshness_unknown" + : liveness.severity === "ok" + ? "fresh" + : liveness.severity === "warn" + ? "lagging" + : "stale"; + return { + type: "liveness", + severity: liveness.severity, + reason, + metricIds: ["liveness"], + recommendation: + liveness.severity === "crit" ? { code: "check_control_plane", link: "status" } : undefined, + }; +} diff --git a/apps/webapp/app/presenters/v3/reports/renderMarkdown.ts b/apps/webapp/app/presenters/v3/reports/renderMarkdown.ts new file mode 100644 index 00000000000..0eb82ab47f6 --- /dev/null +++ b/apps/webapp/app/presenters/v3/reports/renderMarkdown.ts @@ -0,0 +1,522 @@ +/** + * GENERIC renderer: ReportViewModel -> monospace markdown. Severity-driven disclosure: + * a degraded finding expands into evidence in causal order; a healthy finding collapses + * to one `✓` line. Owns ALL presentation (formatting, glyphs, sparklines, spacing, {token} + * substitution) and resolves the VM's codes -> strings via the report's registered message + * catalog, looked up by `vm.title` — so it holds NO report vocabulary itself. + * + * Knows NOTHING about health — walks summary -> findings -> metrics generically. + */ + +import { reportMessages, type ReportMessages } from "./report-messages"; +import { + type Finding, + type FooterEntry, + type Metric, + type ReportViewModel, + type Severity, + type Unit, +} from "./report-view-model"; + +const BARS = "▁▂▃▄▅▆▇█"; +const MINUS = "−"; // U+2212 + +const SEVERITY_GLYPH: Record<Severity, string> = { ok: "✓", warn: "⚠", crit: "✕" }; + +/** + * A NEUTRAL marker for a state that's genuinely unknown, not good/bad — e.g. liveness with no + * telemetry signal. It doesn't affect the aggregate severity (that stays driven by real findings), + * but it must not read as a confident green "✓", so it gets its own glyph. + */ +const NEUTRAL_GLYPH = "○"; + +/** + * Markdown-only status colour. Chat hosts render neither ANSI nor HTML, so swapping + * the glyphs for traffic-light circles is the one colour cue they get — one emoji per + * marker (neutral -> white). ANSI keeps the crisp ✓/⚠/✕/○, so this applies ONLY on markdown. + */ +const MARKDOWN_STATUS_EMOJI: Record<string, string> = { + "✓": "🟢", + "⚠": "🟡", + "✕": "🔴", + "○": "⚪", +}; + +function toMarkdownEmoji(text: string): string { + return text.replace(/[✓⚠✕○]/g, (g) => MARKDOWN_STATUS_EMOJI[g] ?? g); +} + +/** Glyph for a finding/statement: neutral for a genuinely-unknown freshness, else severity-driven. */ +function statusGlyph(severity: Severity, reason?: string): string { + return reason === "freshness_unknown" ? NEUTRAL_GLYPH : SEVERITY_GLYPH[severity]; +} + +/** Evidence lines (metric rows + attribution) shown for a degraded section. */ +const EVIDENCE_CAP = 4; + +/** Column where the header's scope·period·baseline starts (mirrors the mockup). */ +const HEADER_COL = 22; + +/** Gap between aligned columns in an evidence block. */ +const COL_GAP = 3; + +/** Section labels pad to this so their glyph/content aligns vertically. */ +const SECTION_LABEL_WIDTH = 9; // "EXECUTION" +const SECTION_GAP = 3; + +/** Section label padded so every section's ✓ / cause text starts at the same column. */ +function sectionLabel(type: string): string { + return `${type.toUpperCase().padEnd(SECTION_LABEL_WIDTH)}${" ".repeat(SECTION_GAP)}`; +} + +// --------------------------------------------------------------------------- +// Formatters. +// --------------------------------------------------------------------------- + +/** All sparklines render at this fixed width so they align in a column. */ +const SPARK_WIDTH = 8; + +/** Resample to exactly `width` points (bucket-average) so every sparkline aligns. */ +function resampleToWidth(points: number[], width: number): number[] { + if (points.length === 0) return []; + if (points.length === width) return points; + if (points.length < width) { + // stretch: nearest-sample up to width (keeps shape). + return Array.from({ length: width }, (_, i) => points[Math.floor((i * points.length) / width)]); + } + const out: number[] = []; + const stride = points.length / width; + for (let i = 0; i < width; i++) { + const slice = points.slice( + Math.floor(i * stride), + Math.max(Math.floor((i + 1) * stride), Math.floor(i * stride) + 1) + ); + out.push(slice.reduce((a, b) => a + b, 0) / slice.length); + } + return out; +} + +export function sparklineFromSeries(points: number[]): string { + if (points.length === 0) return ""; + const p = resampleToWidth(points, SPARK_WIDTH); + const min = Math.min(...p); + const max = Math.max(...p); + if (max === min) return BARS[0].repeat(SPARK_WIDTH); + return p.map((v) => BARS[Math.round(((v - min) / (max - min)) * (BARS.length - 1))]).join(""); +} + +function fmtCount(n: number): string { + return Math.round(n).toLocaleString("en-US"); +} + +function fmtDuration(ms: number): string { + if (ms < 1000) return `${Math.round(ms)}ms`; + const s = ms / 1000; + if (s < 60) return Number.isInteger(s) ? `${s}s` : `${s.toFixed(1)}s`; + const m = s / 60; + return Number.isInteger(m) ? `${m}m` : `${m.toFixed(1)}m`; +} + +function fmtPct(ratio: number): string { + return `${(ratio * 100).toFixed(1)}%`; +} + +function fmtRate(n: number): string { + return `${fmtCount(n)}/min`; +} + +function fmtSignedRate(net: number): string { + const sign = net < 0 ? MINUS : net > 0 ? "+" : ""; + return `${sign}${fmtCount(Math.abs(net))}/min`; +} + +function fmtValue(value: number, unit: Unit): string { + switch (unit) { + case "ms": + return fmtDuration(value); + case "count": + return fmtCount(value); + case "ratio": + return fmtPct(value); + case "perMin": + return fmtRate(value); + } +} + +function fill(template: string, tokens: Record<string, string | number | undefined>): string { + return template.replace(/\{(\w+)\}/g, (_, k) => { + const v = tokens[k]; + return v === undefined ? `{${k}}` : String(v); + }); +} + +function metricById(metrics: Metric[], id: string): Metric | undefined { + return metrics.find((m) => m.id === id); +} + +// --------------------------------------------------------------------------- +// Metric line (expanded evidence). +// --------------------------------------------------------------------------- + +function deltaSegment(metric: Metric): string { + if (metric.severity === "ok" || !metric.delta || metric.delta.dir === "flat") return ""; + const arrow = metric.delta.dir === "up" ? "↑" : "↓"; + // "up" shows the multiplier when we have it ("↑ 16×"). "down" is arrow-only: a + // drop rounds to 0×/1×, meaningless — the arrow already says "below normal". + if (metric.delta.dir === "down") return arrow; + return metric.delta.mult === undefined ? arrow : `${arrow} ${metric.delta.mult}×`; +} + +function annotationSegment(metric: Metric, vm: ReportViewModel): string { + if (!metric.annotation) return ""; + const value = String(metric.annotation.value ?? ""); + return fill(reportMessages(vm.title).annotationMessage(metric.annotation.code), { + value, + window: vm.windowMinutes, + limit: metric.breakdown?.limit ?? "", + }); +} + +function metricValueText(metric: Metric, msg: ReportMessages): string { + // concurrency etc. carry a limit -> "running/limit". + if (metric.unit === "count" && metric.breakdown?.limit !== undefined) { + return `${fmtCount(metric.value)}/${fmtCount(metric.breakdown.limit)}`; + } + // composite throughput -> "done vs triggered -> net". + if (metric.unit === "perMin" && metric.breakdown?.done !== undefined) { + const { done, triggered } = metric.breakdown; + return `${fmtRate(done)} done vs ${fmtRate(triggered)} triggered → net ${fmtSignedRate(metric.value)}`; + } + const showAgg = + metric.aggregation === "p95" && !msg.metricLabel(metric.id).toLowerCase().startsWith("p95"); + return showAgg + ? `p95 ${fmtValue(metric.value, metric.unit)}` + : fmtValue(metric.value, metric.unit); +} + +/** Column widths for a section's evidence block, so value/delta/spark align down the page. */ +type Cols = { label: number; value: number; delta: number }; + +function isComposite(metric: Metric): boolean { + return metric.unit === "perMin" && metric.breakdown?.done !== undefined; +} + +function computeColumns(rows: Metric[], vm: ReportViewModel, extraLabels: string[]): Cols { + const msg = reportMessages(vm.title); + const labelLens = [ + ...rows.map((m) => msg.metricLabel(m.id).length), + ...extraLabels.map((l) => l.length), + ]; + const valueLens = rows.filter((m) => !isComposite(m)).map((m) => metricValueText(m, msg).length); + const deltaLens = rows + .filter((m) => !isComposite(m) && !annotationSegment(m, vm)) + .map((m) => deltaSegment(m).length); + return { + label: Math.max(0, ...labelLens), + value: Math.max(0, ...valueLens), + delta: Math.max(0, ...deltaLens), + }; +} + +function renderMetricRow(metric: Metric, cols: Cols, vm: ReportViewModel): string { + const msg = reportMessages(vm.title); + const gap = " ".repeat(COL_GAP); + const label = msg.metricLabel(metric.id).padEnd(cols.label); + const value = metricValueText(metric, msg); + + // composite throughput: label + value only (its own grammar). + if (isComposite(metric)) return ` ${label}${gap}${value}`; + + const spark = + metric.series && metric.series.points.length > 0 + ? sparklineFromSeries(metric.series.points) + : ""; + const annotation = annotationSegment(metric, vm); + const delta = annotation ? "" : deltaSegment(metric); // cause line carries an annotation, not a delta + const trailing = annotation + ? annotation + : metric.normal !== undefined + ? `(normal ~${fmtValue(metric.normal, metric.unit)})` + : metric.series?.kind === "estimated" + ? "(estimated)" // proxy trend (e.g. snapshot backlog) — flag it so it isn't read as measured + : ""; + + // Fixed columns: label · value · delta · SPARK · trailing. The fixed-width spark + // column keeps every spark and the trailing (normal / annotation) aligned. + let line = ` ${label}${gap}${value.padEnd(cols.value)}`; + if (cols.delta > 0) line += `${gap}${delta.padEnd(cols.delta)}`; + line += `${gap}${(spark || "").padEnd(SPARK_WIDTH)}`; + if (trailing) line += `${gap}${trailing}`; + return line.replace(/\s+$/, ""); +} + +// --------------------------------------------------------------------------- +// Compact facts (collapsed / semi-expanded healthy sections). +// --------------------------------------------------------------------------- + +function compactFact(metric: Metric): string | undefined { + switch (metric.id) { + case "pending": + return `pending ${fmtCount(metric.value)}${metric.normal !== undefined ? ` (normal ~${fmtCount(metric.normal)})` : ""}`; + case "start_latency_p95": + return `starts p95 ${fmtDuration(metric.value)}`; + case "failures": + return `failures ${fmtPct(metric.value)}${metric.normal !== undefined ? ` (normal ~${fmtPct(metric.normal)})` : ""}`; + case "dur_p95": + return metric.severity === "ok" + ? "durations normal" + : `durations p95 ${fmtDuration(metric.value)}`; + default: + return undefined; // throughput / evidence metrics get no collapsed fact + } +} + +/** Reassuring facts read consequence-first: depth/failures before latency/duration. */ +const COMPACT_ORDER = ["pending", "failures", "start_latency_p95", "dur_p95"]; + +function compactFacts(finding: Finding, metrics: Metric[]): string { + return finding.metricIds + .map((id) => metricById(metrics, id)) + .filter((m): m is Metric => m !== undefined) + .slice() + .sort((a, b) => COMPACT_ORDER.indexOf(a.id) - COMPACT_ORDER.indexOf(b.id)) + .map(compactFact) + .filter((s): s is string => s !== undefined) + .join(" · "); +} + +// --------------------------------------------------------------------------- +// Read / exclusion / attribution / window. +// --------------------------------------------------------------------------- + +function readTokens(_finding: Finding, metrics: Metric[]): Record<string, string | number> { + const triggered = metricById(metrics, "triggered"); + return { + mult: triggered?.delta?.mult ?? "", + }; +} + +function windowSuffix(finding: Finding): string { + const aw = finding.anomalyWindow; + if (!aw) return ""; + return aw.touchesEnd ? ` (last ${aw.minutes} min)` : ` (${aw.minutes} min window)`; +} + +function attributionLine(finding: Finding, cols: Cols): string | undefined { + const a = finding.attribution; + if (!a) return undefined; + const label = `worst ${a.dim}`.padEnd(cols.label); + return ` ${label}${" ".repeat(COL_GAP)}${a.key} — ${Math.round(a.share * 100)}% of ${a.of}`; +} + +// --------------------------------------------------------------------------- +// Sections. +// --------------------------------------------------------------------------- + +function renderDegradedSection(finding: Finding, vm: ReportViewModel): string[] { + const msg = reportMessages(vm.title); + const rows = finding.metricIds + .map((id) => metricById(vm.metrics, id)) + .filter((m): m is Metric => m !== undefined); + + const extraLabels = finding.attribution ? [`worst ${finding.attribution.dim}`] : []; + const cols = computeColumns(rows, vm, extraLabels); + + const evidence: string[] = rows.map((m) => renderMetricRow(m, cols, vm)); + const attr = attributionLine(finding, cols); + if (attr) evidence.push(attr); + + // One blank line between evidence rows so the block breathes. + const spaced = evidence.slice(0, EVIDENCE_CAP).flatMap((l, i) => (i === 0 ? [l] : ["", l])); + + const lines = [ + // Lead with the glyph so the cause text lines up with the healthy sections' "✓ …". + `${sectionLabel(finding.type)}${SEVERITY_GLYPH[finding.severity]} ${msg.findingReason(finding.type, finding.reason)}${windowSuffix(finding)}`, + "", // blank line between the header and its evidence (matches the section/footer spacing) + ...spaced, + ]; + + if (finding.read) { + lines.push( + "", + ` read: ${fill(msg.readMessage(finding.read), readTokens(finding, vm.metrics))}` + ); + // Exclusions ("not your code") first, then supporting observations ("runs completing at ~X/min"). + for (const excl of finding.exclusions ?? []) { + lines.push( + ` ${fill(msg.exclusionMessage(excl.code), { rate: fmtCount(excl.evidence?.donePerMin ?? 0) })}` + ); + } + for (const obs of finding.observations ?? []) { + lines.push( + ` ${fill(msg.observationMessage(obs.code), { rate: fmtCount(obs.evidence?.donePerMin ?? 0) })}` + ); + } + } + return lines; +} + +function renderHealthyExecutionExpanded(finding: Finding, vm: ReportViewModel): string[] { + const msg = reportMessages(vm.title); + const lines = [ + `${sectionLabel(finding.type)}${SEVERITY_GLYPH.ok} ${msg.findingReason(finding.type, finding.reason, { expanded: true })}`, + "", // blank line between the header and its facts (matches the section/footer spacing) + ` ${compactFacts(finding, vm.metrics)}`, + ]; + if (finding.read) lines.push(` read: ${msg.readMessage(finding.read)}`); + return lines; +} + +function renderCollapsedSection(finding: Finding, vm: ReportViewModel): string[] { + const msg = reportMessages(vm.title); + const headline = `${sectionLabel(finding.type)}${SEVERITY_GLYPH.ok} ${msg.findingReason(finding.type, finding.reason)}`; + const facts = compactFacts(finding, vm.metrics); + // Healthy sections stay on one line; the facts are short. + return facts ? [`${headline} — ${facts}`] : [headline]; +} + +function renderLivenessLine(finding: Finding, metrics: Metric[], msg: ReportMessages): string { + const metric = metricById(metrics, finding.metricIds[0]); + const ageMs = metric?.value; + const age = ageMs !== undefined && Number.isFinite(ageMs) ? fmtDuration(ageMs) : "unknown"; + const reason = msg.findingReason(finding.type, finding.reason).replace("{age}", age); + return `${sectionLabel(finding.type)}${statusGlyph(finding.severity, finding.reason)} ${reason}`; +} + +function renderFooter(footer: FooterEntry[], msg: ReportMessages): string[] { + return footer.map((entry, i) => { + const text = fill(msg.actionMessage(entry.code), { value: entry.value, min: entry.value }); + return i === 0 ? `→ ${text}` : ` ${text}`; + }); +} + +// --------------------------------------------------------------------------- +// Top-level render. +// --------------------------------------------------------------------------- + +/** + * The plain monochrome layout (✓/⚠/✕) shared by every surface. Colour renderers paint + * THIS via `paintReport`; markdown swaps the glyphs for emoji. Internal so the three + * public renderers can't drift. + */ +function renderReportPlain(vm: ReportViewModel): string { + const msg = reportMessages(vm.title); + const lines: string[] = []; + + // header: "/report <title>" padded to a column, then scope · period · baseline. + const left = `/report ${vm.title}`; + const right = [vm.scope, vm.period, vm.baselineLabel].filter(Boolean).join(" · "); + lines.push(`${left.padEnd(HEADER_COL)}${right}`, ""); + + // Each statement carries its OWN glyph — one leading glyph would read as if it + // applied only to the first statement (e.g. "✕ Flow healthy"). Per-statement is clear. + const verdict = vm.summary.statements + .map( + (s) => + `${statusGlyph(s.severity, s.reason)} ${msg.statementMessage(s.findingType, s.severity, s.reason)}` + ) + .join(" · "); + lines.push(verdict, ""); + + const flowDegraded = vm.findings.some((f) => f.type === "flow" && f.severity !== "ok"); + + for (const finding of vm.findings) { + if (finding.type === "liveness") { + lines.push(renderLivenessLine(finding, vm.metrics, msg), ""); + } else if (finding.reason === "unknown") { + // stale-data guard: no ✓ or facts computed from a silent feed. The guard forces crit, + // so the glyph reads from severity (consistent with the summary + JSON). + lines.push( + `${sectionLabel(finding.type)}${SEVERITY_GLYPH[finding.severity]} ${msg.findingReason(finding.type, finding.reason)}`, + "" + ); + } else if (finding.severity !== "ok") { + lines.push(...renderDegradedSection(finding, vm), ""); + } else if (finding.type === "execution" && flowDegraded) { + lines.push(...renderHealthyExecutionExpanded(finding, vm), ""); + } else { + lines.push(...renderCollapsedSection(finding, vm), ""); + } + } + + lines.push(...renderFooter(vm.footer, msg)); + + return lines.join("\n").replace(/\n+$/, "\n"); +} + +/** + * Markdown surface (agents / chat). The plain layout with severity glyphs swapped for + * status emoji — the only decoration markdown gets. + */ +export function renderReportMarkdown(vm: ReportViewModel): string { + return toMarkdownEmoji(renderReportPlain(vm)); +} + +// --------------------------------------------------------------------------- +// ANSI colour renderer (terminal, e.g. `trigger report`). Colourises the SAME +// plain layout as a post-pass via `paintReport`, so terminal output can't drift. +// --------------------------------------------------------------------------- + +const SPARK_LOW = "▁▂▃▄▅"; +const SPARK_HIGH = "▆▇█"; + +/** A surface's colour functions. `low`/`high` paint the two sparkline tones. */ +type Paint = { + escape: (s: string) => string; + green: (s: string) => string; + amber: (s: string) => string; + red: (s: string) => string; + grey: (s: string) => string; + low: (s: string) => string; + high: (s: string) => string; +}; + +function paintSpark(run: string, p: Paint): string { + return [...run] + .map((ch) => (SPARK_HIGH.includes(ch) ? p.high(ch) : SPARK_LOW.includes(ch) ? p.low(ch) : ch)) + .join(""); +} + +/** Colourise the plain report by role. Detection reads the RAW line; wrapping the escaped line. */ +function paintReport(text: string, p: Paint): string { + return text + .split("\n") + .map((raw, i) => { + const line = p.escape(raw); + // header: grey the right column (scope · period · baseline). + if (i === 0) { + return line.replace(/^(\/report \S+\s{2,})(.+)$/, (_, a, b) => a + p.grey(b)); + } + // whole-line secondary: read: chain, its exclusion lines, do-nothing footer. + if (/^\s*read:/.test(raw) || /^\s{6,}\S/.test(raw)) return p.grey(line); + if (/^\s{2}(or do nothing|open |Check status)/.test(raw)) return p.grey(line); + + let l = line; + l = l.replace(/[▁▂▃▄▅▆▇█]+/g, (m) => paintSpark(m, p)); // two-tone sparkline + l = l + .replace(/✓/g, p.green("✓")) + .replace(/⚠/g, p.amber("⚠")) + .replace(/✕/g, p.red("✕")) + .replace(/○/g, p.grey("○")); // neutral (unknown) marker + l = l.replace(/↑ ?\d+×|↑/g, (m) => p.amber(m)).replace(/↓ ?\d+×|↓/g, (m) => p.green(m)); + l = l.replace(/\(last \d+ min\)|\(\d+ min window\)/g, (m) => p.amber(m)); // anomaly window + l = l.replace(/\(normal ~[^)]*\)|\(estimated\)/g, (m) => p.grey(m)); // baseline/estimate is context + l = l.replace(/^(\s+worst \w+\s+)(.+?)( — )/, (_, a, k, b) => a + p.green(k) + b); // attribution key (may contain spaces) + return l; + }) + .join("\n"); +} + +const ANSI_PAINT: Paint = { + escape: (s) => s, + green: (s) => `\x1b[32m${s}\x1b[0m`, + amber: (s) => `\x1b[33m${s}\x1b[0m`, + red: (s) => `\x1b[31m${s}\x1b[0m`, + grey: (s) => `\x1b[90m${s}\x1b[0m`, + low: (s) => `\x1b[32m${s}\x1b[0m`, + high: (s) => `\x1b[33m${s}\x1b[0m`, +}; + +export function renderReportAnsi(vm: ReportViewModel): string { + return paintReport(renderReportPlain(vm), ANSI_PAINT); +} diff --git a/apps/webapp/app/presenters/v3/reports/report-messages.ts b/apps/webapp/app/presenters/v3/reports/report-messages.ts new file mode 100644 index 00000000000..2d90ea016ed --- /dev/null +++ b/apps/webapp/app/presenters/v3/reports/report-messages.ts @@ -0,0 +1,39 @@ +/** + * Report-agnostic message infrastructure. Prose lives in each report's OWN catalog (e.g. + * `health/health-messages.ts`); this file only defines the resolver surface + a registry so the + * generic renderer can turn a VM's codes into strings by looking the catalog up via `vm.title`. + * No report vocabulary here — that would re-couple the renderer to a specific report. + */ + +import { type ReasonCode, type Severity } from "./report-view-model"; + +/** + * The resolver surface a report provides. Every renderer resolves a VM's codes through this; + * strings may carry {tokens} (e.g. {age}, {rate}) that the renderer fills from evidence. + */ +export type ReportMessages = { + metricLabel(id: string): string; + findingReason(findingType: string, reason: ReasonCode, opts?: { expanded?: boolean }): string; + readMessage(code: ReasonCode): string; + exclusionMessage(code: ReasonCode): string; + observationMessage(code: ReasonCode): string; + annotationMessage(code: ReasonCode): string; + statementMessage(findingType: string, severity: Severity, reason?: ReasonCode): string; + actionMessage(code: ReasonCode): string; +}; + +const catalogs = new Map<string, ReportMessages>(); + +/** Register a report's catalog under its title (e.g. "health"). Called for its side effect. */ +export function registerReportMessages(title: string, messages: ReportMessages): void { + catalogs.set(title, messages); +} + +/** Look up a report's catalog by `vm.title`. Throws if the report never registered one. */ +export function reportMessages(title: string): ReportMessages { + const messages = catalogs.get(title); + if (!messages) { + throw new Error(`report-messages: no catalog registered for report "${title}"`); + } + return messages; +} diff --git a/apps/webapp/app/presenters/v3/reports/report-registry.ts b/apps/webapp/app/presenters/v3/reports/report-registry.ts new file mode 100644 index 00000000000..a0209dcfeda --- /dev/null +++ b/apps/webapp/app/presenters/v3/reports/report-registry.ts @@ -0,0 +1,37 @@ +/** + * The report catalog: which reports exist and how each loads + interprets its data. Keyed by + * report name so cost/regression/errors drop in later as new `{ load, interpret }` entries with + * no changes to the VM, renderers, route, tool, or presenter. + * + * Deliberately separate from `ReportPresenter` — the presenter only orchestrates (look up a + * loader by key, run it, single-flight); knowing WHICH reports exist is a distinct concern. + */ + +import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { interpret as interpretHealth } from "./health/health"; +import { loadHealthInput } from "./health/health-data"; +import { type ReportViewModel } from "./report-view-model"; + +export type ReportLoader<TInput> = { + load: (env: AuthenticatedEnvironment, period: string) => Promise<TInput>; + interpret: (input: TInput) => ReportViewModel; +}; + +function defineReport<TInput>(loader: ReportLoader<TInput>): ReportLoader<unknown> { + return loader as ReportLoader<unknown>; +} + +export const REPORT_REGISTRY: Record<string, ReportLoader<unknown>> = { + health: defineReport({ + load: (env, period) => loadHealthInput(env, period), + interpret: interpretHealth, + }), +}; + +export const REPORT_KEYS = Object.keys(REPORT_REGISTRY); + +export function isReportKey(key: string): boolean { + // Object.hasOwn, not `in`: `in` matches prototype keys ("toString", "__proto__"), + // which would pass the route guard and then 500 in the loader. + return Object.hasOwn(REPORT_REGISTRY, key); +} diff --git a/apps/webapp/app/presenters/v3/reports/report-view-model.ts b/apps/webapp/app/presenters/v3/reports/report-view-model.ts new file mode 100644 index 00000000000..95ede3a24ca --- /dev/null +++ b/apps/webapp/app/presenters/v3/reports/report-view-model.ts @@ -0,0 +1,226 @@ +/** + * Generic, render-agnostic contract for a Report. Semantic, not a UI tree: numbers + + * what they mean (codes, severities, units, series), never formatted strings or layout. + * Every renderer consumes this and owns presentation. Reasons are codes; + * `report-messages.ts` resolves them -> strings, so phrasing lives in one place. + * + * No React/DOM/IO. Report-agnostic — `health` is just one interpreter that emits it. + */ + +export type Severity = "ok" | "warn" | "crit"; + +export type Unit = "ms" | "count" | "ratio" | "perMin"; + +/** A code resolved to a human string by `report-messages.ts`. */ +export type ReasonCode = string; + +/** A key into `ReportViewModel.links`, so a recommendation can point at a URL. */ +export type LinkKey = string; + +export type Delta = { + dir: "up" | "down" | "flat"; + /** rounded value/normal multiplier; renderer decides whether to print "6×". */ + mult?: number; +}; + +export type MetricSeries = { + points: number[]; + /** "estimated" = a proxy (e.g. pending backlog), shown informational-only. */ + kind: "measured" | "estimated"; +}; + +export type Metric = { + /** CODE, e.g. "start_latency_p95" — messages map -> label "start latency". */ + id: string; + value: number; + unit: Unit; + aggregation?: "p95" | "rate" | "ratio" | "count"; + /** baseline; renderer formats "(normal ~7s)". */ + normal?: number; + delta?: Delta; + series?: MetricSeries; + /** named sub-values for composite metrics (e.g. throughput { done, triggered }). */ + breakdown?: Record<string, number>; + /** shown on a cause line INSTEAD of "(normal ~x)", e.g. "pinned 40 of last 60 min". */ + annotation?: { code: ReasonCode; value?: number }; + /** + * Whether `value` is a real measurement. "unknown" = there was no signal, so `value` is a + * placeholder (e.g. liveness age 0) that a structured consumer must NOT read as a real 0 — + * the finding's reason carries the "unknown" meaning. Absent = measured (the common case). + */ + availability?: "measured" | "unknown"; + severity: Severity; +}; + +export type Recommendation = { + code: ReasonCode; + link?: LinkKey; +}; + +/** A footer line: an action, or the "do nothing" option (carries value). */ +export type FooterEntry = { + code: ReasonCode; + link?: LinkKey; + /** a computed fact (e.g. drainMinutes), never invented. */ + value?: number; +}; + +/** A ruled-out cause + its evidence, e.g. "not your code" (never emitted without evidence). */ +export type Exclusion = { + code: ReasonCode; + evidence?: Record<string, number>; +}; + +/** + * A supporting fact backing the verdict — a measured observation, NOT a ruled-out cause, + * e.g. "runs are completing at ~820/min". Kept separate from `Exclusion` so the two aren't + * conflated (an exclusion answers "what it ISN'T"; an observation states "what IS true"). + */ +export type Observation = { + code: ReasonCode; + evidence?: Record<string, number>; +}; + +export type Finding = { + /** "flow" | "execution" | "liveness" | future "infrastructure" | "billing" */ + type: string; + severity: Severity; + /** CODE for the state/cause, e.g. "env_limit_saturation" | "healthy". */ + reason: ReasonCode; + /** CODE for the "read:" line. Built last, may span findings. */ + read?: ReasonCode; + /** metric ids this finding covers, in causal order when degraded. */ + metricIds: string[]; + /** ONE primary action. */ + recommendation?: Recommendation; + /** optional parenthetical — same shape as recommendation. */ + hedge?: Recommendation; + /** contiguous breach window of the driving metric -> "(last 40 min)". */ + anomalyWindow?: { minutes: number; touchesEnd: boolean }; + /** + * which dimension/key owns the problem, only when share >= threshold. `of` is the + * denominator label the renderer prints (e.g. "pending" for flow, "failures" for execution) + * — so it never mislabels a failures share as "% of pending". + */ + attribution?: { dim: string; key: string; share: number; of: string }; + /** ruled-out causes + evidence — rendered under the `read:` line ("not your code …"). */ + exclusions?: Exclusion[]; + /** supporting facts + evidence — rendered under the `read:` line after the exclusions. */ + observations?: Observation[]; +}; + +export type SummaryStatement = { + findingType: string; + severity: Severity; + /** + * Normally the statement renders from (findingType, severity). Exceptions carry a reason: + * stale telemetry marks flow AND execution "unknown" -> "Flow/Execution unknown — data stale"; + * liveness with no signal is "freshness_unknown" -> "data freshness unknown". + */ + reason?: ReasonCode; +}; + +export type ReportLink = { + key: LinkKey; + label: string; + url: string; +}; + +/** a.k.a. ReportDocument — report-agnostic, render-agnostic. */ +export type ReportViewModel = { + /** "health" | "cost" | … */ + title: string; + /** "prod" */ + scope: string; + /** "last 1h" */ + period: string; + /** "vs your 7d normal" */ + baselineLabel?: string; + /** ISO string — passed in, never read from the clock inside interpret. */ + generatedAt: string; + /** live window length in minutes — lets the renderer say "of last 60 min". */ + windowMinutes: number; + + summary: { + severity: Severity; + statements: SummaryStatement[]; + }; + findings: Finding[]; + metrics: Metric[]; + /** dense structured payload for agents. */ + facts: Record<string, unknown>; + links: ReportLink[]; + /** dominant finding's action + optional "do nothing" option. Max two entries. */ + footer: FooterEntry[]; +}; + +// --------------------------------------------------------------------------- +// Interpret-side helpers (produce VM fields, no prose, no IO). +// --------------------------------------------------------------------------- + +/** Direction + rounded multiplier of `value` against a `normal` baseline. */ +export function delta(value: number, normal: number | undefined): Delta { + if (normal === undefined || !Number.isFinite(normal) || normal === 0) { + return { dir: "flat" }; + } + const diff = value - normal; + const dir = diff > 0 ? "up" : diff < 0 ? "down" : "flat"; + return { dir, mult: Math.round(value / normal) }; +} + +/** Severity of a scalar `x` against ascending warn/crit thresholds. */ +export function classifySeverity(x: number, t: { warn: number; crit: number }): Severity { + if (x >= t.crit) return "crit"; + if (x >= t.warn) return "warn"; + return "ok"; +} + +const SEVERITY_RANK: Record<Severity, number> = { ok: 0, warn: 1, crit: 2 }; + +export function maxSeverity(...severities: Severity[]): Severity { + return severities.reduce<Severity>( + (acc, s) => (SEVERITY_RANK[s] > SEVERITY_RANK[acc] ? s : acc), + "ok" + ); +} + +export function isOk(severity: Severity): boolean { + return severity === "ok"; +} + +/** + * Trailing contiguous run of buckets that breach `threshold`, in minutes over + * `windowMinutes`. Default breach is at/over threshold (ABOVE, e.g. concurrency + * pinned at the limit); `below: true` counts at/under (BELOW, e.g. running capacity + * idle under a stall floor). `touchesEnd` = the run reaches the latest bucket + * ("(last 40 min)" vs mid-window "(14–16h)"). Undefined when nothing breaches. + */ +export function anomalyWindow( + series: number[], + threshold: number, + windowMinutes: number, + options?: { below?: boolean } +): { minutes: number; touchesEnd: boolean } | undefined { + if (series.length === 0) return undefined; + const perBucket = windowMinutes / series.length; + const breaches = options?.below ? (v: number) => v <= threshold : (v: number) => v >= threshold; + + // longest breaching run + whether any run touches the end. + let longest = 0; + let current = 0; + let touchesEnd = false; + for (let i = 0; i < series.length; i++) { + if (breaches(series[i])) { + current++; + longest = Math.max(longest, current); + if (i === series.length - 1) touchesEnd = true; + } else { + current = 0; + } + } + if (longest === 0) return undefined; + // If the run reaches the latest bucket, use the TRAILING length so "(last X min)" is + // accurate — a longer mid-window run must not inflate it. + const runBuckets = touchesEnd ? current : longest; + return { minutes: Math.round(runBuckets * perBucket), touchesEnd }; +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam._index/route.tsx index 99c73f33849..5a30d931c52 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam._index/route.tsx @@ -6,7 +6,7 @@ import type { TaskRunStatus } from "@trigger.dev/database"; import type { PanelHandle } from "@window-splitter/react"; import { Fragment, Suspense, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { ClientOnly } from "remix-utils/client-only"; -import { Bar, BarChart, ReferenceLine, Tooltip, type TooltipProps, YAxis } from "recharts"; +import { Bar, type TooltipProps } from "recharts"; import { TypedAwait, typeddefer, useTypedLoaderData } from "remix-typedjson"; import { BeakerIcon } from "~/assets/icons/BeakerIcon"; import { ClockIcon } from "~/assets/icons/ClockIcon"; @@ -52,6 +52,12 @@ import { } from "~/components/primitives/Table"; import { SimpleTooltip } from "~/components/primitives/Tooltip"; import TooltipPortal from "~/components/primitives/TooltipPortal"; +import { + ActivityBarChart, + ACTIVITY_CHART_HEIGHT, + ACTIVITY_CHART_PEAK_CLASS, + ACTIVITY_CHART_WIDTH, +} from "~/components/metrics/ActivityBarChart"; import { TaskFileName } from "~/components/runs/v3/TaskPath"; import { TaskRunStatusCombo } from "~/components/runs/v3/TaskRunStatus"; import { @@ -626,64 +632,31 @@ const STATUS_BARS: { status: TaskRunStatus; fill: string }[] = [ { status: "TIMED_OUT", fill: "var(--color-run-timed-out)" }, ]; -// Fixed px dims skip ResponsiveContainer's ResizeObserver — otherwise every panel resize re-renders all 25 charts. -const ACTIVITY_CHART_WIDTH = 112; -const ACTIVITY_CHART_HEIGHT = 24; // chart (112) + gap-1.5 (6) + count min-w (28). Reserved so the column stays put while the chart unmounts. const ACTIVITY_CELL_WIDTH = 146; -const ACTIVITY_CHART_COUNT_CLASS = - "-mt-1 inline-block min-w-7 text-xxs tabular-nums text-text-dimmed"; function TaskActivityGraph({ activity }: { activity: HourlyTaskActivity[string] }) { const maxTotal = Math.max(...activity.map((d) => d.total)); return ( - <div className="flex items-start gap-1.5"> - <div - className="rounded-sm" - style={{ width: ACTIVITY_CHART_WIDTH, height: ACTIVITY_CHART_HEIGHT }} - > - <BarChart - data={activity} - width={ACTIVITY_CHART_WIDTH} - height={ACTIVITY_CHART_HEIGHT} - margin={{ top: 0, right: 0, left: 0, bottom: 0 }} - > - <YAxis domain={[0, maxTotal || 1]} hide /> - <Tooltip - cursor={{ fill: "rgba(255, 255, 255, 0.06)" }} - content={<TaskActivityTooltip />} - allowEscapeViewBox={{ x: true, y: true }} - wrapperStyle={{ zIndex: 1000 }} - animationDuration={0} - /> - {STATUS_BARS.map(({ status, fill }) => ( - <Bar - key={status} - dataKey={status} - stackId="a" - fill={fill} - strokeWidth={0} - isAnimationActive={false} - /> - ))} - <ReferenceLine y={0} stroke="var(--color-border-bright)" strokeWidth={1} /> - {maxTotal > 0 && ( - <ReferenceLine - y={maxTotal} - stroke="var(--color-border-brighter)" - strokeDasharray="4 4" - strokeWidth={1} - /> - )} - </BarChart> - </div> - <SimpleTooltip - asChild - button={<span className={ACTIVITY_CHART_COUNT_CLASS}>{formatNumberCompact(maxTotal)}</span>} - content="Peak runs in a single hour" - /> - </div> + <ActivityBarChart + data={activity} + max={maxTotal} + tooltip={<TaskActivityTooltip />} + peak={formatNumberCompact(maxTotal)} + peakTooltip="Peak runs in a single hour" + > + {STATUS_BARS.map(({ status, fill }) => ( + <Bar + key={status} + dataKey={status} + stackId="a" + fill={fill} + strokeWidth={0} + isAnimationActive={false} + /> + ))} + </ActivityBarChart> ); } @@ -701,7 +674,7 @@ function TaskActivityBlankState() { strokeWidth={1} /> </svg> - <span className={ACTIVITY_CHART_COUNT_CLASS}>0</span> + <span className={ACTIVITY_CHART_PEAK_CLASS}>0</span> </div> ); } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx index 6ce526ba7b3..76beaf297f3 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx @@ -6,7 +6,8 @@ import { TypedAwait, typeddefer, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; import { BeakerIcon } from "~/assets/icons/BeakerIcon"; import { CubeSparkleIcon } from "~/assets/icons/CubeSparkleIcon"; -import { PageBody } from "~/components/layout/AppLayout"; +import { PageContainer } from "~/components/layout/AppLayout"; +import { MetricsLayout } from "~/components/layout/MetricsLayout"; import { DirectionSchema, ListPagination } from "~/components/ListPagination"; import { LinkButton } from "~/components/primitives/Buttons"; import { buildActivityTimeAxis } from "~/components/primitives/charts/activityTimeAxis"; @@ -20,11 +21,6 @@ import { Header2 } from "~/components/primitives/Headers"; import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader"; import { Paragraph } from "~/components/primitives/Paragraph"; import * as Property from "~/components/primitives/PropertyTable"; -import { - ResizableHandle, - ResizablePanel, - ResizablePanelGroup, -} from "~/components/primitives/Resizable"; import { Spinner } from "~/components/primitives/Spinner"; import { TabButton, TabContainer } from "~/components/primitives/Tabs"; import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; @@ -45,6 +41,7 @@ import { import { NextRunListPresenter } from "~/presenters/v3/NextRunListPresenter.server"; import { SessionListPresenter } from "~/presenters/v3/SessionListPresenter.server"; import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; +import { getResizableSnapshot } from "~/services/resizablePanel.server"; import { requireUser } from "~/services/session.server"; import { docsPath, @@ -174,6 +171,10 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }) .catch(() => null); + // The [main | config] split is draggable and its width is persisted to a cookie by the + // Resizable primitive; hydrate it here so a reload keeps the user's saved split. + const sidebarSnapshot = await getResizableSnapshot(request, "agent-detail-sidebar"); + return typeddefer({ agent, runActivity, @@ -182,6 +183,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { llmTokenActivity, runList, sessionList, + sidebarSnapshot, }); }; @@ -196,6 +198,7 @@ export default function Page() { llmTokenActivity, runList, sessionList, + sidebarSnapshot, } = useTypedLoaderData<typeof loader>(); const organization = useOrganization(); const project = useProject(); @@ -209,7 +212,7 @@ export default function Page() { const tabLabel = tab === "sessions" ? "Sessions" : "Runs"; return ( - <> + <PageContainer> <NavBar> <PageTitle backButton={{ to: tasksPath, text: "Tasks" }} @@ -230,134 +233,119 @@ export default function Page() { </LinkButton> </PageAccessories> </NavBar> - <PageBody scrollable={false}> - <ResizablePanelGroup orientation="horizontal" className="max-h-full"> - <ResizablePanel id="agent-main" min="300px"> - <div className="grid h-full grid-rows-[auto_1fr] overflow-hidden"> - {/* Top bar — tabs on the left; TimeFilter + pagination on the right. - h-10 matches the right-hand sidebar header height. */} - <div className="flex h-10 items-end border-b border-grid-dimmed bg-background-bright pl-3 pr-2"> - <TabContainer className="-mb-px"> - <TabButton - isActive={tab === "sessions"} - layoutId="agent-page-tabs" - onClick={() => setTab("sessions")} - > - Sessions - </TabButton> - <TabButton - isActive={tab === "runs"} - layoutId="agent-page-tabs" - onClick={() => setTab("runs")} - > - Runs - </TabButton> - </TabContainer> - <div className="ml-auto flex items-center gap-2 self-center"> - <TimeFilter defaultPeriod="7d" labelName={tabLabel} /> - {tab === "sessions" ? ( - <Suspense fallback={null}> - <TypedAwait resolve={sessionList} errorElement={null}> - {(list) => (list ? <ListPagination list={list} /> : null)} - </TypedAwait> - </Suspense> - ) : ( - <Suspense fallback={null}> - <TypedAwait resolve={runList} errorElement={null}> - {(list) => (list ? <ListPagination list={list} /> : null)} - </TypedAwait> - </Suspense> + <MetricsLayout.Root> + {/* Filters — the pinned bar under the NavBar: the TimeFilter and pagination that used to + be fused with the tabs now live here, above the charts (Queues list pattern). Left and + right clusters are child divs; the slot's baked justify-between spreads them. */} + <MetricsLayout.Filters> + <div className="flex items-center gap-2"> + <TimeFilter defaultPeriod="7d" labelName={tabLabel} /> + </div> + <div className="flex items-center gap-2"> + {tab === "sessions" ? ( + <Suspense fallback={null}> + <TypedAwait resolve={sessionList} errorElement={null}> + {(list) => (list ? <ListPagination list={list} /> : null)} + </TypedAwait> + </Suspense> + ) : ( + <Suspense fallback={null}> + <TypedAwait resolve={runList} errorElement={null}> + {(list) => (list ? <ListPagination list={list} /> : null)} + </TypedAwait> + </Suspense> + )} + </div> + </MetricsLayout.Filters> + + {/* Activity / LLM spend / Token charts as a fixed-height chart row (three-up), synced + + drag-to-zoom. The old draggable charts/table split is intentionally dropped — the + charts get a fixed row and the table flows below in the page scroll. */} + <ChartSyncProvider onZoom={zoomToTimeFilter}> + <MetricsLayout.Grid kind="charts" columns={{ base: 1, sm: 3 }}> + <ChartCard title={tabLabel}> + {tab === "sessions" ? ( + <Suspense fallback={<ActivityChartSkeleton />}> + <TypedAwait resolve={sessionActivity} errorElement={<ActivityChartSkeleton />}> + {(result) => <ActivityChart activity={result} />} + </TypedAwait> + </Suspense> + ) : ( + <Suspense fallback={<ActivityChartSkeleton />}> + <TypedAwait resolve={runActivity} errorElement={<ActivityChartSkeleton />}> + {(result) => <ActivityChart activity={result} />} + </TypedAwait> + </Suspense> + )} + </ChartCard> + + <ChartCard title="LLM spend ($)"> + <Suspense fallback={<ActivityChartSkeleton />}> + <TypedAwait resolve={llmCostActivity} errorElement={<ActivityChartSkeleton />}> + {(result) => ( + <ScalarActivityChart + activity={result} + seriesKey="cost" + label="Spend" + color="var(--color-agents)" + valueFormatter={formatCurrency} + /> )} - </div> - </div> - - <ResizablePanelGroup orientation="vertical" className="max-h-full"> - {/* Activity / LLM cost / Token charts */} - <ResizablePanel id="agent-activity" min="220px" default="320px"> - <div className="flex h-full flex-col overflow-hidden bg-background p-2"> - <ChartSyncProvider onZoom={zoomToTimeFilter}> - <div className="grid min-h-0 flex-1 grid-cols-3 gap-2"> - <ChartCard title={tabLabel}> - {tab === "sessions" ? ( - <Suspense fallback={<ActivityChartSkeleton />}> - <TypedAwait - resolve={sessionActivity} - errorElement={<ActivityChartSkeleton />} - > - {(result) => <ActivityChart activity={result} />} - </TypedAwait> - </Suspense> - ) : ( - <Suspense fallback={<ActivityChartSkeleton />}> - <TypedAwait - resolve={runActivity} - errorElement={<ActivityChartSkeleton />} - > - {(result) => <ActivityChart activity={result} />} - </TypedAwait> - </Suspense> - )} - </ChartCard> - - <ChartCard title="LLM spend ($)"> - <Suspense fallback={<ActivityChartSkeleton />}> - <TypedAwait - resolve={llmCostActivity} - errorElement={<ActivityChartSkeleton />} - > - {(result) => ( - <ScalarActivityChart - activity={result} - seriesKey="cost" - label="Spend" - color="var(--color-agents)" - valueFormatter={formatCurrency} - /> - )} - </TypedAwait> - </Suspense> - </ChartCard> - - <ChartCard title="Tokens"> - <Suspense fallback={<ActivityChartSkeleton />}> - <TypedAwait - resolve={llmTokenActivity} - errorElement={<ActivityChartSkeleton />} - > - {(result) => ( - <ScalarActivityChart - activity={result} - seriesKey="tokens" - label="Tokens" - color="#14B8A6" - valueFormatter={formatTokens} - /> - )} - </TypedAwait> - </Suspense> - </ChartCard> - </div> - </ChartSyncProvider> - </div> - </ResizablePanel> - - <ResizableHandle id="agent-activity-handle" /> - - {/* Table */} - <ResizablePanel id="agent-content" min="160px"> - <AgentContentArea tab={tab} sessionList={sessionList} runList={runList} /> - </ResizablePanel> - </ResizablePanelGroup> - </div> - </ResizablePanel> - - <ResizableHandle id="agent-detail-handle" /> - <ResizablePanel id="agent-detail" min="280px" default="380px" max="500px" isStaticAtRest> - <AgentDetailSidebar agent={agent} playgroundPath={playgroundPath} /> - </ResizablePanel> - </ResizablePanelGroup> - </PageBody> - </> + </TypedAwait> + </Suspense> + </ChartCard> + + <ChartCard title="Tokens"> + <Suspense fallback={<ActivityChartSkeleton />}> + <TypedAwait resolve={llmTokenActivity} errorElement={<ActivityChartSkeleton />}> + {(result) => ( + <ScalarActivityChart + activity={result} + seriesKey="tokens" + label="Tokens" + color="#14B8A6" + valueFormatter={formatTokens} + /> + )} + </TypedAwait> + </Suspense> + </ChartCard> + </MetricsLayout.Grid> + </ChartSyncProvider> + + {/* Tabs alone on their row (Queue detail pattern), then the table below them. */} + <MetricsLayout.Content> + <TabContainer className="px-3"> + <TabButton + isActive={tab === "sessions"} + layoutId="agent-page-tabs" + onClick={() => setTab("sessions")} + > + Sessions + </TabButton> + <TabButton + isActive={tab === "runs"} + layoutId="agent-page-tabs" + onClick={() => setTab("runs")} + > + Runs + </TabButton> + </TabContainer> + <AgentContentArea tab={tab} sessionList={sessionList} runList={runList} /> + </MetricsLayout.Content> + + <MetricsLayout.Sidebar + resizable + autosaveId="agent-detail-sidebar" + snapshot={sidebarSnapshot} + min="280px" + defaultSize="380px" + max="500px" + > + <AgentDetailSidebar agent={agent} playgroundPath={playgroundPath} /> + </MetricsLayout.Sidebar> + </MetricsLayout.Root> + </PageContainer> ); } @@ -368,52 +356,44 @@ function AgentContentArea({ sessionList, runList, }: { tab: AgentTab } & Pick<LoaderData, "sessionList" | "runList">) { - return ( - <div className="h-full overflow-hidden"> - {tab === "sessions" ? ( - <Suspense fallback={<TableLoading />}> - <TypedAwait resolve={sessionList} errorElement={<TableLoading />}> - {(list) => - list ? ( - <div className="h-full overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-surface-control"> - <SessionsTable - sessions={list.sessions} - filters={list.filters} - hasFilters={list.hasFilters} - showTopBorder={false} - stickyHeader - /> - </div> - ) : ( - <TableLoading /> - ) - } - </TypedAwait> - </Suspense> - ) : ( - <Suspense fallback={<TableLoading />}> - <TypedAwait resolve={runList} errorElement={<TableLoading />}> - {(list) => - list ? ( - <div className="h-full overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-surface-control"> - <TaskRunsTable - total={list.runs.length} - hasFilters={list.hasFilters} - filters={list.filters} - runs={list.runs} - variant="dimmed" - showTopBorder={false} - stickyHeader - /> - </div> - ) : ( - <TableLoading /> - ) - } - </TypedAwait> - </Suspense> - )} - </div> + // The table flows in the page-level scroll (MetricsLayout.Root scroll="page"); a sticky header + // keeps the column labels pinned as the whole column scrolls. + return tab === "sessions" ? ( + <Suspense fallback={<TableLoading />}> + <TypedAwait resolve={sessionList} errorElement={<TableLoading />}> + {(list) => + list ? ( + <SessionsTable + sessions={list.sessions} + filters={list.filters} + hasFilters={list.hasFilters} + stickyHeader + /> + ) : ( + <TableLoading /> + ) + } + </TypedAwait> + </Suspense> + ) : ( + <Suspense fallback={<TableLoading />}> + <TypedAwait resolve={runList} errorElement={<TableLoading />}> + {(list) => + list ? ( + <TaskRunsTable + total={list.runs.length} + hasFilters={list.hasFilters} + filters={list.filters} + runs={list.runs} + variant="dimmed" + stickyHeader + /> + ) : ( + <TableLoading /> + ) + } + </TypedAwait> + </Suspense> ); } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx index 668244975ab..170a0f49a46 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx @@ -38,6 +38,7 @@ import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstan import { requireUser } from "~/services/session.server"; import { cn } from "~/utils/cn"; import { EnvironmentParamSchema } from "~/utils/pathBuilder"; +import { canAccessQueueMetricsUi } from "~/v3/canAccessQueueMetricsUi.server"; import { QueryScopeSchema } from "~/v3/querySchemas"; import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; import { MetricWidget } from "../resources.metric"; @@ -50,6 +51,15 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const user = await requireUser(request); const { projectParam, organizationSlug, envParam, dashboardKey } = ParamSchema.parse(params); + // The built-in "queues" dashboard is part of the metrics UI (unlinked, but reachable by + // URL), so gate it per-org like the rest of the Queue Metrics view. + if ( + dashboardKey === "queues" && + !(await canAccessQueueMetricsUi({ userId: user.id, organizationSlug })) + ) { + throw new Response(undefined, { status: 404, statusText: "Not found" }); + } + const project = await findProjectBySlug(organizationSlug, projectParam, user.id); if (!project) { throw new Response(undefined, { @@ -376,6 +386,7 @@ export function MetricDashboard({ promptSlugs={prompts.length > 0 ? prompts : undefined} operations={operations.length > 0 ? operations : undefined} providers={providers.length > 0 ? providers : undefined} + fillGaps={widget.fillGaps} config={widget.display} organizationId={organization.id} projectId={project.id} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx index 05b4f4d9b62..17126ad301e 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx @@ -1,9 +1,10 @@ -import { useState } from "react"; +import { useMemo, useState } from "react"; import { Header3 } from "~/components/primitives/Headers"; import { Paragraph } from "~/components/primitives/Paragraph"; import SegmentedControl from "~/components/primitives/SegmentedControl"; import type { QueryScope } from "~/services/queryService.server"; -import { querySchemas } from "~/v3/querySchemas"; +import { listableQuerySchemas } from "~/v3/querySchemas"; +import { useFeatures } from "~/hooks/useFeatures"; import { TryableCodeBlock } from "./TRQLGuideContent"; // Example queries for the Examples tab @@ -211,14 +212,21 @@ LIMIT 20`, }, ]; -const tableOptions = querySchemas.map((s) => ({ label: s.name, value: s.name })); - export function ExamplesContent({ onTryExample, }: { onTryExample: (query: string, scope: QueryScope) => void; }) { - const [selectedTable, setSelectedTable] = useState(querySchemas[0].name); + const { queueMetricsQueryTables } = useFeatures(); + const schemas = useMemo( + () => listableQuerySchemas({ includeQueueMetrics: queueMetricsQueryTables }), + [queueMetricsQueryTables] + ); + const tableOptions = useMemo( + () => schemas.map((s) => ({ label: s.name, value: s.name })), + [schemas] + ); + const [selectedTable, setSelectedTable] = useState(schemas[0].name); const filtered = exampleQueries.filter((e) => e.table === selectedTable); return ( diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/TableSchemaContent.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/TableSchemaContent.tsx index 3ab2a4b41ab..abcc2141f86 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/TableSchemaContent.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/TableSchemaContent.tsx @@ -1,10 +1,11 @@ import type { ColumnSchema } from "@internal/tsql"; -import { useState } from "react"; +import { useMemo, useState } from "react"; import { Badge } from "~/components/primitives/Badge"; import { CopyableText } from "~/components/primitives/CopyableText"; import { Paragraph } from "~/components/primitives/Paragraph"; import SegmentedControl from "~/components/primitives/SegmentedControl"; -import { querySchemas } from "~/v3/querySchemas"; +import { listableQuerySchemas } from "~/v3/querySchemas"; +import { useFeatures } from "~/hooks/useFeatures"; function ColumnHelpItem({ col }: { col: ColumnSchema }) { return ( @@ -43,11 +44,18 @@ function ColumnHelpItem({ col }: { col: ColumnSchema }) { ); } -const tableOptions = querySchemas.map((s) => ({ label: s.name, value: s.name })); - export function TableSchemaContent() { - const [selectedTable, setSelectedTable] = useState(querySchemas[0].name); - const table = querySchemas.find((s) => s.name === selectedTable) ?? querySchemas[0]; + const { queueMetricsQueryTables } = useFeatures(); + const schemas = useMemo( + () => listableQuerySchemas({ includeQueueMetrics: queueMetricsQueryTables }), + [queueMetricsQueryTables] + ); + const tableOptions = useMemo( + () => schemas.map((s) => ({ label: s.name, value: s.name })), + [schemas] + ); + const [selectedTable, setSelectedTable] = useState(schemas[0].name); + const table = schemas.find((s) => s.name === selectedTable) ?? schemas[0]; return ( <div> diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index f8063b812dd..45c395b54f3 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -1,7 +1,7 @@ import { - AdjustmentsHorizontalIcon, ArrowUpCircleIcon, BookOpenIcon, + ExclamationTriangleIcon, PauseIcon, PlayIcon, RectangleStackIcon, @@ -9,9 +9,9 @@ import { import { DialogClose } from "@radix-ui/react-dialog"; import { Form, useNavigation, type MetaFunction } from "@remix-run/react"; import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; -import type { QueueItem } from "@trigger.dev/core/v3/schemas"; import type { RuntimeEnvironmentType } from "@trigger.dev/database"; -import { useEffect, useState } from "react"; +import { useEffect, useMemo, useState, type ReactNode } from "react"; +import { QueuesIcon } from "~/assets/icons/QueuesIcon"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; import { ConcurrencyIcon } from "~/assets/icons/ConcurrencyIcon"; @@ -21,14 +21,13 @@ import { AdminDebugTooltip } from "~/components/admin/debugTooltip"; import { QueuesHasNoTasks } from "~/components/BlankStatePanels"; import { environmentFullTitle } from "~/components/environments/EnvironmentLabel"; import { PageBody, PageContainer } from "~/components/layout/AppLayout"; -import { BigNumber } from "~/components/metrics/BigNumber"; +import { MetricsLayout } from "~/components/layout/MetricsLayout"; import { Badge } from "~/components/primitives/Badge"; -import { Button, LinkButton, type ButtonVariant } from "~/components/primitives/Buttons"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; import { Callout } from "~/components/primitives/Callout"; import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog"; import { FormButtons } from "~/components/primitives/FormButtons"; import { Header3 } from "~/components/primitives/Headers"; -import { Input } from "~/components/primitives/Input"; import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader"; import { PaginationControls } from "~/components/primitives/Pagination"; import { Paragraph } from "~/components/primitives/Paragraph"; @@ -52,6 +51,7 @@ import { TooltipProvider, TooltipTrigger, } from "~/components/primitives/Tooltip"; +import { TasksIcon } from "~/assets/icons/TasksIcon"; import { QueueName } from "~/components/runs/v3/QueueName"; import { env } from "~/env.server"; import { useAutoRevalidate } from "~/hooks/useAutoRevalidate"; @@ -61,9 +61,26 @@ import { useProject } from "~/hooks/useProject"; import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; -import { getUserById } from "~/models/user.server"; import { EnvironmentQueuePresenter } from "~/presenters/v3/EnvironmentQueuePresenter.server"; import { QueueListPresenter } from "~/presenters/v3/QueueListPresenter.server"; +import { + QueueMetricsPresenter, + type QueueListMetric, +} from "~/presenters/v3/QueueMetricsPresenter.server"; +import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; +import { useSearchParams } from "~/hooks/useSearchParam"; +import { parseFiniteInt } from "~/utils/searchParams"; +import { MiniLineChart } from "~/components/metrics/MiniLineChart"; +import { buildActivityTimeAxis } from "~/components/primitives/charts/activityTimeAxis"; +import { Chart, type ChartConfig } from "~/components/primitives/charts/ChartCompound"; +import { ChartCard } from "~/components/primitives/charts/ChartCard"; +import { ChartSyncProvider } from "~/components/primitives/charts/ChartSyncContext"; +import { useZoomToTimeFilter } from "~/hooks/useZoomToTimeFilter"; +import { + useMetricResourceQuery, + type MetricResourceTimeRange, +} from "~/hooks/useMetricResourceQuery"; +import { logger } from "~/services/logger.server"; import { requireUserId } from "~/services/session.server"; import { cn } from "~/utils/cn"; import { ENVIRONMENT_PAUSE_SOURCE_BILLING_LIMIT } from "~/utils/environmentPauseSource"; @@ -72,18 +89,43 @@ import { docsPath, EnvironmentParamSchema, v3BillingPath, + v3QueuePath, v3RunsPath, } from "~/utils/pathBuilder"; -import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server"; import { PauseEnvironmentService } from "~/v3/services/pauseEnvironment.server"; -import { PauseQueueService } from "~/v3/services/pauseQueue.server"; +import { handleQueueMutationAction } from "~/models/queueMutation.server"; +import { + QueueOverrideConcurrencyButton, + QueuePauseResumeButton, +} from "~/components/queues/QueueControls"; import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; +import { BigNumber } from "~/components/metrics/BigNumber"; +import { canAccessQueueMetricsUi } from "~/v3/canAccessQueueMetricsUi.server"; +import { QueueAllocationPresenter } from "~/presenters/v3/QueueAllocationPresenter.server"; const SearchParamsSchema = z.object({ query: z.string().optional(), page: z.coerce.number().min(1).default(1), + period: z.string().optional(), + from: z.string().optional(), + to: z.string().optional(), + sort: z.enum(["busiest", "queued", "name"]).optional(), }); +const QUEUE_METRICS_DEFAULT_PERIOD = "1d"; + +// The live "Queued" / "Running" header blocks poll ClickHouse on a short cadence so they stay +// current after first paint. They read the env-wide gauges from env_metrics (the env-level rollup +// of queue_metrics, cheapest for a dimension-free query), always over a fixed 15m window regardless +// of the chart/table period, and are NOT scoped to the visible queue set (the blocks are env-wide). +const QUEUE_LIVE_BLOCKS_PERIOD = "15m"; +const QUEUE_LIVE_BLOCKS_QUERY = + "SELECT timeBucket() AS t, max(max_env_queued) AS env_queued, max(max_env_running) AS env_running FROM env_metrics GROUP BY t ORDER BY t"; +// Trust the ClickHouse gauge only while its newest bucket is this recent; otherwise fall back to +// the loader's Redis-exact live values (matches LIVE_GAUGE_FRESH_MS on the queue detail page / run +// inspector). +const LIVE_GAUGE_FRESH_MS = 90_000; + export const meta: MetaFunction = () => { return [ { @@ -97,7 +139,9 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); const url = new URL(request.url); - const { page, query } = SearchParamsSchema.parse(Object.fromEntries(url.searchParams)); + const { page, query, period, from, to, sort } = SearchParamsSchema.parse( + Object.fromEntries(url.searchParams) + ); const project = await findProjectBySlug(organizationSlug, projectParam, userId); if (!project) { @@ -115,22 +159,86 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }); } + // Per-org gate for the metrics UI. When off, this org gets the classic Queues page and + // no metrics query fires. + const queueMetricsUiEnabled = await canAccessQueueMetricsUi({ userId, organizationSlug }); + try { const queueListPresenter = new QueueListPresenter(); const queues = await queueListPresenter.call({ environment, query, page, + // Relevance ordering rides the metrics pipeline, so it is part of the gated UI. + sort: queueMetricsUiEnabled ? (sort ?? "busiest") : "name", }); const environmentQueuePresenter = new EnvironmentQueuePresenter(); const autoReloadPollIntervalMs = env.QUEUES_AUTORELOAD_POLL_INTERVAL_MS; + // Per-queue list metrics (Delay p95 + backlog sparkline columns) are SSR'd with the table. + // The environment header tiles are fetched client-side per card (see QueueEnvMetricChart) so a + // slow ClickHouse query never blocks the queues list from rendering. + let metrics: { + bucketStartMs: number; + bucketIntervalMs: number; + byQueue: Record<string, QueueListMetric>; + } | null = null; + + if (queueMetricsUiEnabled && queues.success) { + // Metrics are additive observability; a ClickHouse hiccup must not take down queue + // management. Fail open to metrics: null instead of bubbling to the page-level 400. + try { + const presenter = new QueueMetricsPresenter(); + const queueNames = queues.queues.map((q) => + q.type === "task" ? `task/${q.name}` : q.name + ); + const timeRange = timeFilterFromTo({ + period, + from: parseFiniteInt(from), + to: parseFiniteInt(to), + defaultPeriod: QUEUE_METRICS_DEFAULT_PERIOD, + }); + const queueMetrics = + queueNames.length > 0 + ? await presenter.getQueueListMetrics({ + environment, + queueNames, + from: timeRange.from, + to: timeRange.to, + }) + : null; + if (queueMetrics) { + metrics = { + bucketStartMs: queueMetrics.bucketStartMs, + bucketIntervalMs: queueMetrics.bucketIntervalMs, + byQueue: Object.fromEntries(queueMetrics.byQueue), + }; + } + } catch (error) { + logger.warn("Queue list metrics unavailable, rendering without them", { error }); + } + } + + // Allocation summary (Environment limit + Allocated tiles) is additive; a presenter + // failure must not 400 the page, so fail open to null like the metrics block above. + let allocation: Awaited<ReturnType<QueueAllocationPresenter["call"]>> | null = null; + if (queueMetricsUiEnabled && queues.success) { + try { + allocation = await new QueueAllocationPresenter().call({ environment }); + } catch (error) { + logger.warn("Queue allocation summary unavailable, rendering without it", { error }); + } + } + return typedjson({ ...queues, environment: await environmentQueuePresenter.call(environment), autoReloadPollIntervalMs, + metrics, + allocation, + queueMetricsUiEnabled, }); } catch (error) { console.error(error); @@ -179,6 +287,19 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { return redirectWithErrorMessage(redirectPath, request, "This branch is archived"); } + // Per-queue actions (pause/resume/override/remove-override) are shared with the queue detail + // route, so they live in a helper that both routes call. + const queueMutation = await handleQueueMutationAction({ + request, + environment, + userId, + formData, + redirectPath, + }); + if (queueMutation) { + return queueMutation; + } + switch (action) { case "environment-pause": { const pauseService = new PauseEnvironmentService(); @@ -196,109 +317,1296 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { } return redirectWithSuccessMessage(redirectPath, request, "Environment resumed"); } - case "queue-pause": - case "queue-resume": { - const friendlyId = formData.get("friendlyId"); - if (!friendlyId) { - return redirectWithErrorMessage(redirectPath, request, "Queue ID is required"); - } + default: + return redirectWithErrorMessage(redirectPath, request, "Something went wrong"); + } +}; - const queueService = new PauseQueueService(); - const result = await queueService.call( - environment, - friendlyId.toString(), - action === "queue-pause" ? "paused" : "resumed" - ); +// Derives the environment concurrency status ("limit" | "burst" | "within") and the matching +// text color from the current running count vs. the env limit and burst factor. Shared by both +// the classic and metrics views so the "Running" tile styling stays in sync. +function getEnvConcurrencyLimitStatus(environment: { + running: number; + concurrencyLimit: number; + burstFactor: number; +}) { + const limitStatus = + environment.running === environment.concurrencyLimit * environment.burstFactor + ? "limit" + : environment.running > environment.concurrencyLimit + ? "burst" + : "within"; - if (!result.success) { - return redirectWithErrorMessage( - redirectPath, - request, - result.error ?? `Failed to ${action === "queue-pause" ? "pause" : "resume"} queue` - ); - } + const limitClassName = + limitStatus === "burst" ? "text-warning" : limitStatus === "limit" ? "text-error" : undefined; - return redirectWithSuccessMessage( - redirectPath, - request, - `Queue ${action === "queue-pause" ? "paused" : "resumed"}` - ); + return { limitStatus, limitClassName }; +} + +export default function Page() { + // Per-org flag decides which whole page renders. Off => the classic Queues page, + // byte-for-byte the pre-metrics UI. Each branch is its own component (own hooks). + const { queueMetricsUiEnabled } = useTypedLoaderData<typeof loader>(); + return queueMetricsUiEnabled ? <QueuesWithMetricsView /> : <ClassicQueuesView />; +} + +function QueuesWithMetricsView() { + const { + environment, + queues, + success, + pagination, + code, + totalQueues, + hasFilters, + autoReloadPollIntervalMs, + metrics, + allocation, + } = useTypedLoaderData<typeof loader>(); + + const metricsByQueue = metrics?.byQueue ?? {}; + + // The four header charts mirror exactly the queue set the table is showing (post-search, + // post-pagination). These are the `task/`-prefixed queue_name values the queue_metrics table + // stores, so the client-side tile queries scope to the same rows the loader listed. + const chartQueueNames = success + ? queues.map((q) => (q.type === "task" ? `task/${q.name}` : q.name)) + : []; + + const organization = useOrganization(); + const project = useProject(); + const env = useEnvironment(); + const plan = useCurrentPlan(); + // Queue metrics are retained for 30 days in ClickHouse, so cap the picker there even for + // plans whose query-period limit was raised above it — a longer window would render empty. + const planPeriodDays = plan?.v3Subscription?.plan?.limits?.queryPeriodDays?.number; + const maxPeriodDays = Math.min(planPeriodDays ?? 30, 30); + + // The header tiles fetch client-side with the same period/from/to the TimeFilter writes. + const { value } = useSearchParams(); + const timeRange = { + period: value("period") ?? null, + from: value("from") ?? null, + to: value("to") ?? null, + }; + + useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true }); + + // Drag-to-zoom on either chart narrows the page's from/to search params, which the + // TimeFilter and the client-side metric queries both read (same wiring as the Agent page). + const zoomToTimeFilter = useZoomToTimeFilter(); + + // Live env-wide Queued/Running blocks. First paint uses the loader's Redis-exact values; from the + // first poll on we prefer ClickHouse so the blocks stay current without a full page revalidate. + // Empty rows (quiet env, or the very first fetch still in flight) fall back to the loader values, + // so we never flash a stale 0. Fixed 15m window, env-wide (no queue filter), CH-only recurring + // load; pauses while the tab is hidden (handled inside the hook). + const { rows: liveBlockRows } = useMetricResourceQuery(QUEUE_LIVE_BLOCKS_QUERY, { + organizationId: organization.id, + projectId: project.id, + environmentId: env.id, + timeRange: { period: QUEUE_LIVE_BLOCKS_PERIOD, from: null, to: null }, + defaultPeriod: QUEUE_LIVE_BLOCKS_PERIOD, + fillGaps: false, + refreshIntervalMs: 15_000, + }); + const lastLiveBlockRow = + liveBlockRows.length > 0 ? liveBlockRows[liveBlockRows.length - 1] : null; + // Only trust the gauge while its newest bucket is fresh. A row painted from the hook's cache on + // client-side nav-back (responseCache), or a quiet env whose latest bucket is minutes old, must + // not override the loader's Redis-exact live values with a stale count. + const lastLiveBucketMs = lastLiveBlockRow ? tileTimeToMs(lastLiveBlockRow.t) : NaN; + const freshLiveBlockRow = + lastLiveBlockRow && + Number.isFinite(lastLiveBucketMs) && + Date.now() - lastLiveBucketMs < LIVE_GAUGE_FRESH_MS + ? lastLiveBlockRow + : null; + const envQueuedLive = freshLiveBlockRow + ? tileNumber(freshLiveBlockRow.env_queued) + : environment.queued; + const envRunningLive = freshLiveBlockRow + ? tileNumber(freshLiveBlockRow.env_running) + : environment.running; + + // Allocation summary tiles. The presenter computes the env-wide allocated total (sum of + // each queue's explicit limit clamped to the env limit) in a single aggregate query. + const envLimit = environment.concurrencyLimit; + const burstLimit = Math.round(envLimit * environment.burstFactor); + const allocated = allocation?.allocated ?? 0; + const allocationPct = envLimit > 0 ? Math.round((allocated / envLimit) * 100) : 0; + const overAllocated = allocated > envLimit; + + // Running-block tinting (burst/limit) tracks the live running value, not the loader snapshot. + const { limitStatus, limitClassName } = getEnvConcurrencyLimitStatus({ + running: envRunningLive, + concurrencyLimit: environment.concurrencyLimit, + burstFactor: environment.burstFactor, + }); + + // Client-side, header-click sorting over the current page's rows. Server pagination and the + // default busiest order are unchanged; clearing a sort returns to that server order. + const queueRows = queues ?? []; + + return ( + <PageContainer> + <NavBar> + <PageTitle title="Queues" /> + <PageAccessories> + <AdminDebugTooltip /> + <LinkButton + variant={"docs/small"} + LeadingIcon={BookOpenIcon} + to={docsPath("/queue-concurrency")} + > + Queues docs + </LinkButton> + </PageAccessories> + </NavBar> + <MetricsLayout.Root> + {/* Filters — pinned bar directly under the NavBar. Left cluster = search + period; right + cluster = pagination. */} + {success ? ( + <MetricsLayout.Filters className="px-2"> + <div className="flex items-center gap-1.5"> + <QueueFilters /> + </div> + <div className="flex items-center gap-1.5"> + <TimeFilter + defaultPeriod={QUEUE_METRICS_DEFAULT_PERIOD} + labelName="Period" + maxPeriodDays={maxPeriodDays} + shortcut={{ key: "d" }} + /> + {environment.runsEnabled && + env.pauseSource !== ENVIRONMENT_PAUSE_SOURCE_BILLING_LIMIT ? ( + <EnvironmentPauseResumeButton env={env} /> + ) : null} + <PaginationControls + currentPage={pagination.currentPage} + totalPages={pagination.mode === "unfiltered" ? pagination.totalPages : 1} + hasNextPage={pagination.mode === "filtered" ? pagination.hasMore : undefined} + showPageNumbers={false} + /> + </div> + </MetricsLayout.Filters> + ) : null} + + {/* Queued + Running + Allocated + Environment limit summary. Four stat tiles: the grid + derives its columns from the tile count (two-up, four-up from lg). The allocation + presenter fails open to null (a ClickHouse/PG hiccup mustn't take down the tiles), so + only the Allocated tile depends on it — the other three + controls always render, and + Allocated shows a "–" placeholder to keep the 4-tile grid shape stable. */} + {success ? ( + <MetricsLayout.Grid> + <BigNumber + title="Queued" + value={envQueuedLive} + suffix={env.paused ? <span className="text-warning">paused</span> : undefined} + animate + accessory={ + <span className="opacity-0 transition-opacity group-focus-within:opacity-100 group-hover:opacity-100"> + <LinkButton + variant="minimal/small" + className="aspect-square px-1!" + LeadingIcon={RunsIcon} + leadingIconClassName="text-text-dimmed group-hover/button:text-text-bright" + to={v3RunsPath(organization, project, env, { + statuses: ["PENDING"], + period: "30d", + rootOnly: false, + })} + tooltip="View queued runs" + /> + </span> + } + valueClassName={env.paused ? "text-warning tabular-nums" : "tabular-nums"} + compactThreshold={1000000} + /> + <BigNumber + title="Running" + value={envRunningLive} + animate + valueClassName={cn(limitClassName, "tabular-nums")} + suffix={ + limitStatus === "burst" ? ( + <span className={cn(limitClassName, "flex items-center gap-1")}> + Including {envRunningLive - environment.concurrencyLimit} burst runs{" "} + <BurstFactorTooltip environment={environment} /> + </span> + ) : limitStatus === "limit" ? ( + "At concurrency limit" + ) : undefined + } + accessory={ + <span className="opacity-0 transition-opacity group-focus-within:opacity-100 group-hover:opacity-100"> + <LinkButton + variant="minimal/small" + className="aspect-square px-1!" + LeadingIcon={RunsIcon} + leadingIconClassName="text-text-dimmed group-hover/button:text-text-bright" + to={v3RunsPath(organization, project, env, { + statuses: ["DEQUEUED", "EXECUTING"], + period: "30d", + rootOnly: false, + })} + tooltip="View in-progress runs" + /> + </span> + } + compactThreshold={1000000} + /> + <BigNumber + title={ + <span className="flex items-center gap-1"> + Allocated + {allocation && overAllocated ? ( + <InfoIconTooltip + content="The queue limits add up to more than the environment limit, so queues will compete for concurrency when the environment saturates." + buttonClassName="text-warning" + /> + ) : null} + </span> + } + value={allocation ? allocated : undefined} + formattedValue={allocation ? undefined : "–"} + valueClassName={cn(allocation && overAllocated && "text-warning")} + suffix={allocation ? `${allocationPct}% of the environment limit` : undefined} + suffixClassName="text-text-dimmed" + /> + <BigNumber + title="Environment limit" + value={envLimit} + suffix={environment.burstFactor > 1 ? `bursts up to ${burstLimit}` : undefined} + suffixClassName="text-text-dimmed" + accessory={ + plan ? ( + plan?.v3Subscription?.plan?.limits.concurrentRuns.canExceed ? ( + <LinkButton + to={concurrencyPath(organization, project, env)} + variant="secondary/small" + LeadingIcon={ConcurrencyIcon} + leadingIconClassName="text-amber-500" + > + Increase limit + </LinkButton> + ) : ( + <LinkButton + to={v3BillingPath(organization, "Upgrade your plan for more concurrency")} + variant="secondary/small" + LeadingIcon={ArrowUpCircleIcon} + leadingIconClassName="text-indigo-500" + > + Increase limit + </LinkButton> + ) + ) : undefined + } + /> + </MetricsLayout.Grid> + ) : null} + + {/* Env saturation, Backlog, Scheduling delay p95, Throttled viz — full-size, synced, + drag-to-zoom line charts (Agent page pattern). Four chart tiles: 2x2 below lg, 4-up + from lg, derived from the tile count. `kind="charts"` bakes the fixed row height. + Only when there are queues to chart: not-success states (engine-version, no tasks) and a + filtered-to-empty list leave chartQueueNames empty, where the tiles would just render + four "No activity" cards above the blank state. */} + {chartQueueNames.length > 0 ? ( + <ChartSyncProvider onZoom={zoomToTimeFilter}> + <MetricsLayout.Grid kind="charts"> + {QUEUE_HEADER_TILES.map((tile) => ( + <QueueEnvMetricChart + key={tile.id} + tile={tile} + timeRange={timeRange} + queueNames={chartQueueNames} + referenceLines={ + tile.id === "saturation" + ? [ + { + y: 100, + label: `Limit ${environment.concurrencyLimit}`, + labelPlacement: "outside" as const, + }, + ...(environment.burstFactor > 1 + ? [ + { + y: Math.round(environment.burstFactor * 100), + label: `Burst ${Math.round( + environment.concurrencyLimit * environment.burstFactor + )}`, + labelPlacement: "outside" as const, + }, + ] + : []), + ] + : undefined + } + // Saturation recolours the line above its 100% limit with a gradient split, so + // only the portion over the line is orange (the offset is derived from the line's + // own value range, so the split lands exactly at 100% regardless of domain + // padding). p95 and throttled use a per-bucket overlay: it retraces only the + // over-threshold stretches, so under-threshold buckets stay blue. + thresholdStroke={ + tile.id === "saturation" + ? { value: 100, aboveColor: "var(--color-warning)" } + : undefined + } + warningOverlay={ + tile.id === "p95" + ? { threshold: 60_000 } + : tile.id === "throttled" + ? // Integer counts: threshold 0 warns once a bucket has ≥1 throttle. + { threshold: 0 } + : undefined + } + /> + ))} + </MetricsLayout.Grid> + </ChartSyncProvider> + ) : null} + + {success ? ( + <MetricsLayout.Content> + {/* Default overflow-x-auto container so wide tables still scroll horizontally on + narrow viewports; the page (not this region) owns vertical scrolling. */} + <Table containerClassName="border-t"> + <TableHeader> + <TableRow> + <TableHeaderCell>Name</TableHeaderCell> + <TableHeaderCell alignment="right">Queued</TableHeaderCell> + <TableHeaderCell alignment="right">Running</TableHeaderCell> + <TableHeaderCell alignment="right">Limit</TableHeaderCell> + <TableHeaderCell + alignment="right" + tooltipContentClassName="max-w-max" + disableTooltipHoverableContent + tooltip={ + <div className="max-w-max space-y-1 p-1 text-left text-xs text-text-dimmed"> + <p> + <span className="text-text-bright">Environment</span>: uses the + environment limit of {environment.concurrencyLimit}. + </p> + <p> + <span className="text-text-bright">User</span>: a limit you set in your + code. + </p> + <p> + <span className="text-text-bright">Override</span>: a limit you set here + or via the API. + </p> + </div> + } + > + Limited by + </TableHeaderCell> + <TableHeaderCell alignment="right">Health</TableHeaderCell> + <TableHeaderCell + alignment="right" + disableTooltipHoverableContent + tooltip="How long runs waited before starting (95% were faster), over the selected time." + > + Delay p95 + </TableHeaderCell> + <TableHeaderCell + alignment="right" + disableTooltipHoverableContent + tooltip={ + <> + How many runs were waiting, over the selected time. <WarningSwatch /> marks + where the queue was throttled. + </> + } + > + Backlog + </TableHeaderCell> + <TableHeaderCell className="w-[1%] pl-32"> + <span className="sr-only">Pause/resume</span> + </TableHeaderCell> + </TableRow> + </TableHeader> + <TableBody> + {queueRows.length > 0 ? ( + queueRows.map((queue) => { + const limit = queue.concurrencyLimit ?? environment.concurrencyLimit; + const isAtConcurrencyLimit = queue.running >= limit; + const isAtQueueLimit = + environment.queueSizeLimit !== null && + queue.queued >= environment.queueSizeLimit; + const queueFilterableName = queueMetricsKey(queue); + const queueMetric = metricsByQueue[queueFilterableName]; + const queueDetailPath = v3QueuePath(organization, project, env, { + friendlyId: queue.id, + }); + return ( + <TableRow key={queue.name}> + <TableCell + to={queueDetailPath} + isTabbableCell + // The queue-type icon and the at-limit warning are real <button>s, so + // they render beside the link (leading/trailing), never inside it — + // otherwise the cell is invalid <a><button> nesting. The name stays the + // link. (Override state is already explained by the "Limited by" column.) + leadingContent={ + <SimpleTooltip + button={ + queue.type === "task" ? ( + <TasksIcon + className={cn( + "size-[1.125rem] text-blue-500", + queue.paused && "opacity-50" + )} + /> + ) : ( + <QueuesIcon + className={cn( + "size-[1.125rem] text-purple-500", + queue.paused && "opacity-50" + )} + /> + ) + } + content={ + queue.type === "task" + ? `This queue was automatically created from your "${queue.name}" task` + : "This is a custom queue you added in your code." + } + /> + } + trailingContent={ + isAtConcurrencyLimit ? ( + <SimpleTooltip + button={<ExclamationTriangleIcon className="size-4 text-warning" />} + content="At concurrency limit: this queue is running as many runs as its limit allows; new runs wait in the backlog." + className="max-w-[230px]" + disableHoverableContent + /> + ) : null + } + > + <span className="flex items-center gap-2"> + <span className={queue.paused ? "opacity-50" : undefined}> + {queue.name} + </span> + {queue.paused ? ( + <Badge variant="extra-small" className="text-warning"> + Paused + </Badge> + ) : null} + {isAtQueueLimit ? ( + <Badge variant="extra-small" className="text-error"> + At queue limit + </Badge> + ) : null} + </span> + </TableCell> + <TableCell + to={queueDetailPath} + alignment="right" + actionClassName="pl-16 tabular-nums" + className={cn( + "w-[1%]", + queue.paused ? "opacity-50" : undefined, + isAtQueueLimit && "text-error" + )} + > + {queue.queued} + </TableCell> + <TableCell + to={queueDetailPath} + alignment="right" + actionClassName="pl-16 tabular-nums" + className={cn( + "w-[1%]", + queue.paused ? "opacity-50" : undefined, + queue.running > 0 && "text-text-bright" + )} + > + {queue.running} + </TableCell> + <TableCell + to={queueDetailPath} + alignment="right" + actionClassName="pl-16 tabular-nums" + className={cn( + "w-[1%]", + queue.paused ? "opacity-50" : undefined, + queue.concurrency?.overriddenAt && "font-medium text-text-bright" + )} + > + {queue.concurrencyLimitOverridePercent !== null ? ( + <> + {limit} + <span className="ml-1 text-text-dimmed group-hover/table-row:text-text-bright"> + ({formatOverridePercent(queue.concurrencyLimitOverridePercent)}%) + </span> + </> + ) : ( + limit + )} + </TableCell> + <TableCell + to={queueDetailPath} + alignment="right" + actionClassName="pl-16" + className={cn("w-[1%]", queue.paused ? "opacity-50" : undefined)} + // Keep the whole row navigable: the override explainer is a tooltip + // button, so it renders beside the link (trailing) rather than nested + // inside the <a>, and the label itself stays the link. + trailingContent={ + queue.concurrency?.overriddenAt ? ( + <InfoIconTooltip + content={ + queue.concurrencyLimitOverridePercent !== null + ? `Overridden at ${formatOverridePercent( + queue.concurrencyLimitOverridePercent + )}% of the environment limit.` + : `This queue's concurrency limit has been manually overridden to ${limit}.` + } + contentClassName="max-w-[230px]" + disableHoverableContent + // Tighten the gap from the "Override" label to gap-1 (the cell's + // trailing adornment gap is gap-2; -ml-1 pulls the icon in 4px). + buttonClassName="-ml-1" + /> + ) : undefined + } + > + {queue.concurrency?.overriddenAt + ? "Override" + : queue.concurrencyLimit + ? "User" + : "Environment"} + </TableCell> + <TableCell + to={queueDetailPath} + alignment="right" + className={cn(queue.paused ? "opacity-50" : undefined)} + > + <QueueHealthBadge + paused={queue.paused} + running={queue.running} + queued={queue.queued} + limit={limit} + /> + </TableCell> + <TableCell + to={queueDetailPath} + alignment="right" + actionClassName="pl-16 tabular-nums" + className="w-[1%]" + > + {queueMetric && queueMetric.p95WaitMs !== null ? ( + <span className="text-text-bright"> + {formatWaitMs(queueMetric.p95WaitMs)} + </span> + ) : ( + <span className="text-text-dimmed">–</span> + )} + </TableCell> + <TableCell to={queueDetailPath} alignment="right"> + <MiniLineChart + data={queueMetric?.depthSparkline} + throttled={queueMetric?.throttledSparkline} + peak={queueMetric?.peakQueued} + bucketStartMs={metrics?.bucketStartMs} + bucketIntervalMs={metrics?.bucketIntervalMs} + width={134} + color="var(--color-queues)" + unitLabel={{ singular: "queued", plural: "queued" }} + showPeak={false} + formatPeak={(v) => v.toLocaleString()} + peakTooltip={ + queueMetric && queueMetric.throttledTotal > 0 + ? `Peak queued; this queue was throttled ${queueMetric.throttledTotal.toLocaleString()} ${ + queueMetric.throttledTotal === 1 ? "time" : "times" + } in this period` + : "Peak queued in this period" + } + /> + </TableCell> + <TableCellMenu + isSticky + visibleButtons={queue.paused && <QueuePauseResumeButton queue={queue} />} + hiddenButtons={!queue.paused && <QueuePauseResumeButton queue={queue} />} + popoverContent={ + <> + {queue.paused ? ( + <QueuePauseResumeButton + queue={queue} + variant="minimal/small" + fullWidth + showTooltip={false} + /> + ) : ( + <QueuePauseResumeButton + queue={queue} + variant="minimal/small" + fullWidth + showTooltip={false} + /> + )} + + <PopoverMenuItem + icon={RunsIcon} + leadingIconClassName="text-runs size-[1.125rem]" + title="View all runs" + to={v3RunsPath(organization, project, env, { + queues: [queueFilterableName], + period: "30d", + rootOnly: false, + })} + /> + <PopoverMenuItem + icon={QueuesIcon} + leadingIconClassName="text-queues size-[1.125rem]" + title="View queued runs" + to={v3RunsPath(organization, project, env, { + queues: [queueFilterableName], + statuses: ["PENDING"], + period: "30d", + rootOnly: false, + })} + /> + <PopoverMenuItem + icon={Spinner} + leadingIconClassName="text-queues animate-none" + title="View in-progress runs" + to={v3RunsPath(organization, project, env, { + queues: [queueFilterableName], + statuses: ["DEQUEUED", "EXECUTING"], + period: "30d", + rootOnly: false, + })} + /> + <QueueOverrideConcurrencyButton + queue={queue} + environmentConcurrencyLimit={environment.concurrencyLimit} + /> + </> + } + /> + </TableRow> + ); + }) + ) : ( + <TableRow> + <TableCell colSpan={9}> + <div className="grid place-items-center py-6 text-text-dimmed"> + <Paragraph> + {hasFilters ? "No queues found matching your filters" : "No queues found"} + </Paragraph> + </div> + </TableCell> + </TableRow> + )} + </TableBody> + </Table> + </MetricsLayout.Content> + ) : ( + <div className="grid place-items-center py-6 text-text-dimmed"> + {totalQueues === 0 ? ( + <div className="pt-12"> + <QueuesHasNoTasks /> + </div> + ) : code === "engine-version" ? ( + <EngineVersionUpgradeCallout /> + ) : ( + <Callout variant="error">Something went wrong</Callout> + )} + </div> + )} + </MetricsLayout.Root> + </PageContainer> + ); +} + +function EnvironmentPauseResumeButton({ + env, +}: { + env: { type: RuntimeEnvironmentType; paused: boolean }; +}) { + const navigation = useNavigation(); + const [isOpen, setIsOpen] = useState(false); + + useEffect(() => { + if (navigation.state === "loading" || navigation.state === "idle") { + setIsOpen(false); } - case "queue-override": { - const friendlyId = formData.get("friendlyId"); - const concurrencyLimit = formData.get("concurrencyLimit"); + }, [navigation.state]); - if (!friendlyId) { - return redirectWithErrorMessage(redirectPath, request, "Queue ID is required"); - } + const isLoading = Boolean( + navigation.formData?.get("action") === (env.paused ? "environment-resume" : "environment-pause") + ); - if (!concurrencyLimit) { - return redirectWithErrorMessage(redirectPath, request, "Concurrency limit is required"); - } + return ( + <Dialog open={isOpen} onOpenChange={setIsOpen}> + <div> + <TooltipProvider disableHoverableContent={true}> + <Tooltip> + <TooltipTrigger asChild> + <div className="cursor-pointer [&_button]:cursor-pointer"> + <DialogTrigger asChild> + <Button + type="button" + variant="secondary/small" + LeadingIcon={env.paused ? PlayIcon : PauseIcon} + leadingIconClassName={env.paused ? "text-success" : "text-warning"} + className={ + env.paused + ? "border-success/60 text-success [&_span]:text-success hover:border-success" + : "border-warning/60 text-warning [&_span]:text-warning hover:border-warning" + } + aria-label={ + env.paused + ? `Resumes ${environmentFullTitle(env)} so its runs can be dequeued again.` + : `Pauses all runs from being dequeued in ${environmentFullTitle(env)}. Any executing runs will continue to run.` + } + > + {env.paused + ? `Resume ${environmentFullTitle(env)} environment…` + : `Pause ${environmentFullTitle(env)} environment…`} + </Button> + </DialogTrigger> + </div> + </TooltipTrigger> + <TooltipContent className={"text-xs"}> + {env.paused + ? `Resumes ${environmentFullTitle(env)} so its runs can be dequeued again.` + : `Pauses all runs from being dequeued in ${environmentFullTitle(env)}. Any executing runs will continue to run.`} + </TooltipContent> + </Tooltip> + </TooltipProvider> + </div> + <DialogContent> + <DialogHeader>{env.paused ? "Resume environment?" : "Pause environment?"}</DialogHeader> + <div className="flex flex-col gap-3 pt-3"> + <Paragraph> + {env.paused + ? `This will allow runs to be dequeued in ${environmentFullTitle(env)} again.` + : `This will pause all runs from being dequeued in ${environmentFullTitle( + env + )}. Any executing runs will continue to run.`} + </Paragraph> + <Form method="post" onSubmit={() => setIsOpen(false)}> + <input + type="hidden" + name="action" + value={env.paused ? "environment-resume" : "environment-pause"} + /> + <FormButtons + confirmButton={ + <Button + type="submit" + disabled={isLoading} + variant={env.paused ? "primary/medium" : "danger/medium"} + LeadingIcon={ + isLoading ? <Spinner color="white" /> : env.paused ? PlayIcon : PauseIcon + } + shortcut={{ modifiers: ["mod"], key: "enter" }} + > + {env.paused ? "Resume environment" : "Pause environment"} + </Button> + } + cancelButton={ + <DialogClose asChild> + <Button type="button" variant="secondary/medium"> + Cancel + </Button> + </DialogClose> + } + /> + </Form> + </div> + </DialogContent> + </Dialog> + ); +} - const limitNumber = parseInt(concurrencyLimit.toString(), 10); - if (isNaN(limitNumber) || limitNumber < 0) { - return redirectWithErrorMessage( - redirectPath, - request, - "Concurrency limit must be a valid number" - ); - } +function EngineVersionUpgradeCallout() { + return ( + <div className="mt-4 flex max-w-lg flex-col gap-4 rounded-sm border border-grid-bright bg-background-bright px-4"> + <div className="flex items-center justify-between gap-2 border-b border-grid-dimmed py-4"> + <h4 className="text-base text-text-bright">New queues table</h4> + <LinkButton + LeadingIcon={BookOpenIcon} + to={docsPath("upgrade-to-v4")} + variant={"docs/small"} + > + Upgrade guide + </LinkButton> + </div> + <div className="space-y-4 pb-4"> + <Paragraph variant="small"> + Upgrade to SDK version 4+ to view the new queues table, and be able to pause and resume + individual queues. + </Paragraph> + <img + src={upgradeForQueuesPath} + alt="Upgrade for queues" + className="rounded-sm border border-grid-dimmed" + /> + </div> + </div> + ); +} + +export function isEnvironmentPauseResumeFormSubmission( + formMethod: string | undefined, + formData: FormData | undefined +) { + if (!formMethod || !formData) { + return false; + } + + return ( + formMethod.toLowerCase() === "post" && + (formData.get("action") === "environment-pause" || + formData.get("action") === "environment-resume") + ); +} + +export function QueueFilters() { + return <SearchInput placeholder="Search queues…" paramName="query" resetParams={["page"]} />; +} + +type MetricTileRow = Record<string, number | string | null>; + +/** One charted point per time bucket, already aggregated across the visible queue set. */ +type TilePoint = { bucket: number; value: number }; + +// Inline colour swatch matching the chart's warning ("yellow") line — used in tooltip copy that +// refers to that colour instead of naming it, so the swatch always matches the chart. +function WarningSwatch() { + return ( + <span + className="mx-0.5 inline-block size-2.5 -translate-y-px rounded-[2px] align-middle" + style={{ backgroundColor: "var(--color-warning)" }} + aria-label="yellow" + /> + ); +} + +type QueueHeaderTile = { + id: string; + label: string; + /** Info-icon copy explaining what the chart shows, rendered next to the card title. */ + description: ReactNode; + color: string; + /** Optional inline legend rendered below the card title: a fixed set of {colored square, label} + * entries, for charts where a colour (e.g. the orange warning line) needs explaining. */ + legend?: Array<{ color: string; label: string }>; + query: string; + /** Formats a single bucket's value in the chart tooltip. */ + formatValue?: (value: number) => string; + /** Formats the y-axis tick labels. Without it the axis shows raw numbers (bad for durations + * in ms or percent scales). Passed through to Chart.Line's yAxisProps.tickFormatter. */ + formatAxis?: (value: number) => string; + /** Hover tooltip explaining the headline readout next to the title (e.g. what "9% of current + * period" means). Without it the readout has no tooltip. */ + totalTooltip?: string; + // Rows can be one-per-bucket (p95, throttled: aggregated across the set in ClickHouse) or + // one-per-(bucket, queue) (saturation, backlog: summed across the set here, since summing a + // gauge across queues can't be a flat aggregate without double-counting sub-buckets). Either + // way derive returns the per-bucket points the chart draws. + derive: (rows: MetricTileRow[]) => { + points: TilePoint[]; + total: number; + formatTotal?: (total: number) => string; + totalClassName?: string; + }; +}; + +function tileNumber(value: number | string | null): number { + const n = typeof value === "number" ? value : Number(value); + return Number.isFinite(n) ? n : 0; +} + +function tileTimeToMs(value: number | string | null): number { + const s = String(value).replace(" ", "T"); + return Date.parse(s.endsWith("Z") ? s : `${s}Z`); +} + +// Sums a per-(bucket, queue) row set into one value per bucket. `read` pulls the queue's +// contribution; `envColumn`, when set, carries an env-wide column (identical across the set's +// rows in a bucket) through as the max, so saturation can divide by the env limit. +function sumByBucket( + rows: MetricTileRow[], + read: (row: MetricTileRow) => number, + envColumn?: string +): Array<{ bucket: number; sum: number; env: number }> { + const byBucket = new Map<number, { sum: number; env: number }>(); + for (const row of rows) { + const bucket = tileTimeToMs(row.t); + if (!Number.isFinite(bucket)) continue; + const entry = byBucket.get(bucket) ?? { sum: 0, env: 0 }; + entry.sum += read(row); + if (envColumn) entry.env = Math.max(entry.env, tileNumber(row[envColumn])); + byBucket.set(bucket, entry); + } + return [...byBucket.entries()] + .map(([bucket, { sum, env }]) => ({ bucket, sum, env })) + .sort((a, b) => a.bucket - b.bucket); +} + +// Header tiles fetch their own TRQL query client-side (resources.metric) with fillGaps, scoped to +// the visible queue set (queue_metrics WHERE queue IN <set>). Saturation and backlog GROUP BY the +// queue too and sum here; p95 merges quantile states and throttled sums counters in ClickHouse. +const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ + { + id: "saturation", + label: "Env saturation", + description: ( + <> + How much of the environment's concurrency these queues are using. Turns <WarningSwatch />{" "} + above 100%, when they're into burst capacity. + </> + ), + color: "var(--color-queues)", + legend: [ + { color: "var(--color-queues)", label: "Saturation" }, + { color: "var(--color-warning)", label: "Over limit" }, + ], + // Numerator: running summed across the visible set. Denominator: the env-wide limit (same for + // every queue in a bucket), so the line reads as the set's share of the environment capacity. + query: `SELECT timeBucket() AS t,\n queue,\n max(max_running) AS running,\n max(max_env_limit) AS env_limit\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + formatValue: (v) => (v > 100 ? `${v}% — over the environment limit` : `${v}%`), + formatAxis: (v) => `${v}%`, + derive: (rows) => { + const points = sumByBucket(rows, (r) => tileNumber(r.running), "env_limit").map( + ({ bucket, sum, env }) => ({ + bucket, + value: env > 0 ? Math.round((sum / env) * 100) : 0, + }) + ); + const peak = points.reduce((max, p) => Math.max(max, p.value), 0); + return { points, total: peak, formatTotal: (v) => `${v}% peak` }; + }, + }, + { + id: "backlog", + label: "Backlog", + description: "How many runs are waiting across these queues, over time.", + color: "var(--color-queues)", + query: `SELECT timeBucket() AS t,\n queue,\n max(max_queued) AS queued\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + derive: (rows) => { + const points = sumByBucket(rows, (r) => tileNumber(r.queued)).map(({ bucket, sum }) => ({ + bucket, + value: sum, + })); + const peak = points.reduce((max, p) => Math.max(max, p.value), 0); + return { points, total: peak, formatTotal: (v) => `${v.toLocaleString()} peak` }; + }, + }, + { + id: "p95", + label: "Scheduling delay p95", + description: ( + <> + How long runs wait before they start (95% start faster than this). Turns <WarningSwatch />{" "} + above 1 minute. + </> + ), + totalTooltip: "The worst p95 in the selected window.", + color: "var(--color-queues)", + legend: [ + { color: "var(--color-queues)", label: "p95" }, + { color: "var(--color-warning)", label: "Over 1 min" }, + ], + // quantilesMerge over the set's rows in a bucket is the true p95 across the union of samples + // (merging quantile states is valid; averaging per-queue percentiles would not be). + query: `SELECT timeBucket() AS t,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95\nFROM queue_metrics\nGROUP BY t\nORDER BY t`, + formatValue: formatWaitMs, + formatAxis: formatWaitMs, + derive: (rows) => { + const points = rows.map((r) => ({ bucket: tileTimeToMs(r.t), value: tileNumber(r.p95) })); + const worst = points.reduce((max, p) => Math.max(max, p.value), 0); + return { + points, + total: worst, + formatTotal: (v) => (v > 0 ? formatWaitMs(v) : "–"), + totalClassName: worst >= 60_000 ? "text-warning" : undefined, + }; + }, + }, + { + id: "throttled", + label: "Throttled", + description: "How often runs were held back by a limit.", + totalTooltip: "The share of the selected window with at least one blocked dequeue.", + color: "var(--color-queues)", + legend: [{ color: "var(--color-warning)", label: "Throttled" }], + query: `SELECT timeBucket() AS t,\n sum(throttled_count) AS throttled\nFROM queue_metrics\nGROUP BY t\nORDER BY t`, + derive: (rows) => { + const points = rows.map((r) => ({ + bucket: tileTimeToMs(r.t), + value: tileNumber(r.throttled), + })); + // Share of the window that saw any throttling. A raw event sum isn't interpretable (it + // scales with poll rate and window length); the fraction of buckets with a throttle is. + // The data path fills gaps (zero-fill for this counter), so every bucket in the window is + // present and `points.length` is the honest denominator. + const nonzero = points.filter((p) => p.value > 0).length; + const pct = points.length > 0 ? Math.round((nonzero / points.length) * 100) : 0; + return { + points, + total: pct, + formatTotal: (v) => `${v}% of current period`, + totalClassName: pct > 0 ? "text-warning" : undefined, + }; + }, + }, +]; + +// When a search matches no queues the set is empty. We still fetch (hooks can't be conditional), +// but with a queue name that can't exist so the IN filter returns nothing and the tile falls +// through to its "No activity" empty state instead of silently widening to the whole environment. +const NO_QUEUES_SENTINEL = "__no_queues__"; + +type TileTimeRange = MetricResourceTimeRange; + +// Full-size env metric chart rendered inside a ChartCard. Same data path as before +// (client-side TRQL via useMetricResourceQuery with fillGaps), drawn as a line that +// participates in the shared hover + drag-to-zoom of the enclosing ChartSyncProvider. +function QueueEnvMetricChart({ + tile, + timeRange, + queueNames, + referenceLines, + thresholdStroke, + warningOverlay, + solidWarning = false, +}: { + tile: QueueHeaderTile; + timeRange: TileTimeRange; + /** The visible queue set (post-search, post-pagination) the chart scopes to. */ + queueNames: string[]; + referenceLines?: Array<{ + y: number; + label?: string; + labelPlacement?: "inside" | "outside"; + }>; + thresholdStroke?: { value: number; aboveColor: string }; + warningOverlay?: { threshold: number }; + /** When set, the ENTIRE line turns warning-coloured if the series is ever non-zero (used for + * throttling: any throttle in the window colours the whole line). Mutually exclusive with the + * per-bucket warningOverlay. */ + solidWarning?: boolean; +}) { + const organization = useOrganization(); + const project = useProject(); + const environment = useEnvironment(); + + // Scope to exactly the queues the table is showing. Empty set => sentinel that matches nothing, + // so the tile shows "No activity" rather than the whole environment. The hook re-fetches when + // this list changes (it keys on the joined names), so search/pagination reflow the charts. + const { rows, showLoading, failed } = useMetricResourceQuery(tile.query, { + organizationId: organization.id, + projectId: project.id, + environmentId: environment.id, + timeRange, + defaultPeriod: QUEUE_METRICS_DEFAULT_PERIOD, + fillGaps: true, + queues: queueNames.length > 0 ? queueNames : [NO_QUEUES_SENTINEL], + }); + + const { points, total, formatTotal, totalClassName } = tile.derive(rows); + + // Same point shape the shared axis/tooltip helpers expect. + const data = points + .map((p) => ({ bucket: p.bucket, [tile.id]: p.value })) + .filter((p) => Number.isFinite(p.bucket)); + + // Whole-line warning colour when the series was ever non-zero (throttling: one throttle in the + // window colours the entire line). Otherwise the tile's normal colour. + const wholeLineWarning = solidWarning && total > 0; + const lineColor = wholeLineWarning ? "var(--color-warning)" : tile.color; + + const chartConfig = useMemo<ChartConfig>( + () => ({ [tile.id]: { label: tile.label, color: lineColor } }), + [tile.id, tile.label, lineColor] + ); + + const { tickFormatter, tooltipLabelFormatter } = useMemo( + () => buildActivityTimeAxis(data), + [data] + ); + const hasData = data.length > 0 && data.some((p) => (p[tile.id] as number) > 0); + + // Peak readout lives in the card title (ChartCard has no dedicated value slot). A zero/empty + // total renders no readout at all (skipping "0% peak", "0 peak", "0" and the p95 "–" placeholder) + // so the card title stands alone until there's a non-zero value to show. + const peak = showLoading ? ( + <span className="inline-block h-3 w-12 animate-pulse rounded bg-grid-bright" /> + ) : failed || total === 0 ? null : formatTotal ? ( + formatTotal(total) + ) : ( + total.toLocaleString() + ); - const user = await getUserById(userId); - if (!user) { - return redirectWithErrorMessage(redirectPath, request, "User not found"); + return ( + <ChartCard + title={ + <span className="flex flex-col gap-1"> + <span className="flex items-baseline gap-2"> + <span className="flex items-center gap-1"> + {tile.label} + <InfoIconTooltip + content={tile.description} + contentClassName="max-w-[230px]" + disableHoverableContent + /> + </span> + {peak != null ? ( + tile.totalTooltip && !showLoading ? ( + <SimpleTooltip + button={ + <span + className={cn( + "text-xs font-normal tabular-nums text-text-dimmed", + totalClassName + )} + > + {peak} + </span> + } + content={tile.totalTooltip} + className="max-w-[230px]" + disableHoverableContent + /> + ) : ( + <span + className={cn( + "text-xs font-normal tabular-nums text-text-dimmed", + totalClassName + )} + > + {peak} + </span> + ) + ) : null} + </span> + {tile.legend && (showLoading || hasData) ? ( + <span className="flex items-center gap-2"> + {tile.legend.map((item) => ( + <span + key={item.label} + className="flex items-center gap-1 text-xs font-normal text-text-dimmed" + > + <span + className="size-2.5 rounded-[2px]" + style={{ backgroundColor: item.color }} + /> + {item.label} + </span> + ))} + </span> + ) : null} + </span> } + > + {showLoading ? ( + <QueueMetricChartSkeleton /> + ) : failed ? ( + <div className="flex h-full items-center justify-center text-xs text-text-dimmed"> + Unable to load metrics + </div> + ) : hasData ? ( + <Chart.Root + config={chartConfig} + data={data} + dataKey="bucket" + series={[tile.id]} + fillContainer + > + <Chart.Line + showDots={false} + referenceLines={referenceLines} + thresholdStroke={thresholdStroke} + warningOverlay={warningOverlay} + xAxisProps={{ tickFormatter }} + yAxisProps={tile.formatAxis ? { tickFormatter: tile.formatAxis } : undefined} + tooltipLabelFormatter={tooltipLabelFormatter} + tooltipValueFormatter={tile.formatValue} + /> + </Chart.Root> + ) : ( + <div className="flex h-full items-center justify-center text-xs text-text-dimmed"> + No activity + </div> + )} + </ChartCard> + ); +} + +function QueueMetricChartSkeleton() { + return ( + <div className="flex h-full min-h-0 items-end gap-px rounded-sm"> + {Array.from({ length: 42 }).map((_, i) => ( + <div key={i} className="h-full flex-1 bg-background-dimmed" /> + ))} + </div> + ); +} + +/** Health as a stock Badge: color carries the state, w-fit keeps it content-width. */ +type QueueHealth = { + paused: boolean; + running: number; + queued: number; + limit: number; +}; - const result = await concurrencySystem.queues.overrideQueueConcurrencyLimit( - environment, - friendlyId.toString(), - limitNumber, - user - ); +type QueueHealthLabel = "Paused" | "At capacity" | "Backlogged" | "Active" | "Idle"; - if (!result.isOk()) { - return redirectWithErrorMessage( - redirectPath, - request, - "Failed to override queue concurrency limit" - ); - } +// Single source of truth for the queue health decision, shared by the badge and the table's +// health-column sort so the sorted order always matches the labels shown. +function queueHealthLabel({ paused, running, queued, limit }: QueueHealth): QueueHealthLabel { + if (paused) return "Paused"; + if (running >= limit && queued > 0) return "At capacity"; + if (queued > 0) return "Backlogged"; + if (running > 0) return "Active"; + return "Idle"; +} - return redirectWithSuccessMessage( - redirectPath, - request, - "Queue concurrency limit overridden" - ); - } - case "queue-remove-override": { - const friendlyId = formData.get("friendlyId"); +const QUEUE_HEALTH_STYLES: Record<QueueHealthLabel, string> = { + Paused: "text-warning", + "At capacity": "text-warning", + Backlogged: "text-blue-500", + Active: "text-success", + Idle: "text-text-dimmed", +}; - if (!friendlyId) { - return redirectWithErrorMessage(redirectPath, request, "Queue ID is required"); - } +function QueueHealthBadge(health: QueueHealth) { + const label = queueHealthLabel(health); + return ( + <Badge variant="extra-small" className={cn("ml-auto w-fit", QUEUE_HEALTH_STYLES[label])}> + {label} + </Badge> + ); +} - const result = await concurrencySystem.queues.resetConcurrencyLimit( - environment, - friendlyId.toString() - ); +// The `queue_metrics`-prefixed key a queue is stored under (task queues are prefixed `task/`). +function queueMetricsKey(queue: { type: string; name: string }): string { + return `${queue.type === "task" ? "task/" : ""}${queue.name}`; +} - if (!result.isOk()) { - return redirectWithErrorMessage( - redirectPath, - request, - "Failed to reset queue concurrency limit" - ); - } +function formatWaitMs(ms: number): string { + if (ms < 1000) return `${Math.round(ms)}ms`; + if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`; + if (ms < 3_600_000) return `${(ms / 60_000).toFixed(1)}m`; + return `${(ms / 3_600_000).toFixed(1)}h`; +} - return redirectWithSuccessMessage(redirectPath, request, "Queue concurrency limit reset"); - } - default: - return redirectWithErrorMessage(redirectPath, request, "Something went wrong"); - } -}; +// Drop a trailing ".00" from whole percentages so "50.00" reads as "50" but "12.50" is preserved. +function formatOverridePercent(percent: number): string { + return Number.isInteger(percent) ? percent.toString() : percent.toFixed(2).replace(/\.?0+$/, ""); +} -export default function Page() { +// Classic Queues page, restored verbatim from before the Queue Metrics feature. Rendered +// when queueMetricsUiEnabled is off so a gated org sees exactly the pre-metrics UI. +function ClassicQueuesView() { const { environment, queues, @@ -317,15 +1625,7 @@ export default function Page() { useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true }); - const limitStatus = - environment.running === environment.concurrencyLimit * environment.burstFactor - ? "limit" - : environment.running > environment.concurrencyLimit - ? "burst" - : "within"; - - const limitClassName = - limitStatus === "burst" ? "text-warning" : limitStatus === "limit" ? "text-error" : undefined; + const { limitStatus, limitClassName } = getEnvConcurrencyLimitStatus(environment); return ( <PageContainer> @@ -531,7 +1831,7 @@ export default function Page() { </Badge> } content="This queue's concurrency limit has been manually overridden from the dashboard or API." - className="max-w-xs" + className="max-w-[230px]" disableHoverableContent /> ) : null} @@ -703,362 +2003,6 @@ export default function Page() { ); } -function EnvironmentPauseResumeButton({ - env, -}: { - env: { type: RuntimeEnvironmentType; paused: boolean }; -}) { - const navigation = useNavigation(); - const [isOpen, setIsOpen] = useState(false); - - useEffect(() => { - if (navigation.state === "loading" || navigation.state === "idle") { - setIsOpen(false); - } - }, [navigation.state]); - - const isLoading = Boolean( - navigation.formData?.get("action") === (env.paused ? "environment-resume" : "environment-pause") - ); - - return ( - <Dialog open={isOpen} onOpenChange={setIsOpen}> - <div> - <TooltipProvider disableHoverableContent={true}> - <Tooltip> - <TooltipTrigger asChild> - <div> - <DialogTrigger asChild> - <Button - type="button" - variant="secondary/small" - LeadingIcon={env.paused ? PlayIcon : PauseIcon} - leadingIconClassName={env.paused ? "text-success" : "text-warning"} - > - {env.paused ? "Resume..." : "Pause environment..."} - </Button> - </DialogTrigger> - </div> - </TooltipTrigger> - <TooltipContent className={"text-xs"}> - {env.paused - ? `Resume processing runs in ${environmentFullTitle(env)}` - : `Pause processing runs in ${environmentFullTitle(env)}`} - </TooltipContent> - </Tooltip> - </TooltipProvider> - </div> - <DialogContent> - <DialogHeader>{env.paused ? "Resume environment?" : "Pause environment?"}</DialogHeader> - <div className="flex flex-col gap-3 pt-3"> - <Paragraph> - {env.paused - ? `This will allow runs to be dequeued in ${environmentFullTitle(env)} again.` - : `This will pause all runs from being dequeued in ${environmentFullTitle( - env - )}. Any executing runs will continue to run.`} - </Paragraph> - <Form method="post" onSubmit={() => setIsOpen(false)}> - <input - type="hidden" - name="action" - value={env.paused ? "environment-resume" : "environment-pause"} - /> - <FormButtons - confirmButton={ - <Button - type="submit" - disabled={isLoading} - variant={env.paused ? "primary/medium" : "danger/medium"} - LeadingIcon={ - isLoading ? <Spinner color="white" /> : env.paused ? PlayIcon : PauseIcon - } - shortcut={{ modifiers: ["mod"], key: "enter" }} - > - {env.paused ? "Resume environment" : "Pause environment"} - </Button> - } - cancelButton={ - <DialogClose asChild> - <Button type="button" variant="tertiary/medium"> - Cancel - </Button> - </DialogClose> - } - /> - </Form> - </div> - </DialogContent> - </Dialog> - ); -} - -function QueuePauseResumeButton({ - queue, - variant = "tertiary/small", - fullWidth = false, - showTooltip = true, -}: { - /** The "id" here is a friendlyId */ - queue: { id: string; name: string; paused: boolean }; - variant?: ButtonVariant; - fullWidth?: boolean; - showTooltip?: boolean; -}) { - const [isOpen, setIsOpen] = useState(false); - - const trigger = showTooltip ? ( - <div> - <TooltipProvider disableHoverableContent={true}> - <Tooltip> - <TooltipTrigger asChild> - <div> - <DialogTrigger asChild> - <Button - type="button" - variant={variant} - LeadingIcon={queue.paused ? PlayIcon : PauseIcon} - leadingIconClassName={queue.paused ? "text-success" : "text-warning"} - fullWidth={fullWidth} - textAlignLeft={fullWidth} - > - {queue.paused ? "Resume..." : "Pause..."} - </Button> - </DialogTrigger> - </div> - </TooltipTrigger> - <TooltipContent side="right" className={"text-xs"}> - {queue.paused - ? `Resume processing runs in queue "${queue.name}"` - : `Pause processing runs in queue "${queue.name}"`} - </TooltipContent> - </Tooltip> - </TooltipProvider> - </div> - ) : ( - <DialogTrigger asChild> - <PopoverMenuItem - icon={queue.paused ? PlayIcon : PauseIcon} - leadingIconClassName={queue.paused ? "text-success" : "text-warning"} - title={queue.paused ? "Resume..." : "Pause..."} - /> - </DialogTrigger> - ); - - return ( - <Dialog open={isOpen} onOpenChange={setIsOpen}> - {trigger} - <DialogContent> - <DialogHeader>{queue.paused ? "Resume queue?" : "Pause queue?"}</DialogHeader> - <div className="flex flex-col gap-3 pt-3"> - <Paragraph> - {queue.paused - ? `This will allow runs to be dequeued in the "${queue.name}" queue again.` - : `This will pause all runs from being dequeued in the "${queue.name}" queue. Any executing runs will continue to run.`} - </Paragraph> - <Form method="post" onSubmit={() => setIsOpen(false)}> - <input - type="hidden" - name="action" - value={queue.paused ? "queue-resume" : "queue-pause"} - /> - <input type="hidden" name="friendlyId" value={queue.id} /> - <FormButtons - confirmButton={ - <Button - type="submit" - shortcut={{ modifiers: ["mod"], key: "enter" }} - variant={queue.paused ? "primary/medium" : "danger/medium"} - LeadingIcon={queue.paused ? PlayIcon : PauseIcon} - > - {queue.paused ? "Resume queue" : "Pause queue"} - </Button> - } - cancelButton={ - <DialogClose asChild> - <Button type="button" variant="tertiary/medium"> - Cancel - </Button> - </DialogClose> - } - /> - </Form> - </div> - </DialogContent> - </Dialog> - ); -} - -function QueueOverrideConcurrencyButton({ - queue, - environmentConcurrencyLimit, -}: { - queue: QueueItem; - environmentConcurrencyLimit: number; -}) { - const navigation = useNavigation(); - const [isOpen, setIsOpen] = useState(false); - const [concurrencyLimit, setConcurrencyLimit] = useState<string>( - queue.concurrencyLimit?.toString() ?? environmentConcurrencyLimit.toString() - ); - - const isOverridden = !!queue.concurrency?.overriddenAt; - const currentLimit = queue.concurrencyLimit ?? environmentConcurrencyLimit; - - useEffect(() => { - if (navigation.state === "loading" || navigation.state === "idle") { - setIsOpen(false); - } - }, [navigation.state]); - - const isLoading = Boolean( - navigation.formData?.get("action") === "queue-override" || - navigation.formData?.get("action") === "queue-remove-override" - ); - - return ( - <Dialog open={isOpen} onOpenChange={setIsOpen}> - <DialogTrigger asChild> - <PopoverMenuItem - icon={AdjustmentsHorizontalIcon} - title={isOverridden ? "Edit override…" : "Override limit…"} - /> - </DialogTrigger> - <DialogContent> - <DialogHeader> - {isOverridden ? "Edit concurrency override" : "Override concurrency limit"} - </DialogHeader> - <div className="flex flex-col gap-3 pt-3"> - {isOverridden ? ( - <Paragraph> - This queue's concurrency limit is currently overridden to {currentLimit}. - {typeof queue.concurrency?.base === "number" && - ` The original limit set in code was ${queue.concurrency.base}.`}{" "} - You can update the override or remove it to restore the{" "} - {typeof queue.concurrency?.base === "number" - ? "limit set in code" - : "environment concurrency limit"} - . - </Paragraph> - ) : ( - <Paragraph> - Override this queue's concurrency limit. The current limit is {currentLimit}, which is - set {queue.concurrencyLimit !== null ? "in code" : "by the environment"}. - </Paragraph> - )} - <Form method="post" onSubmit={() => setIsOpen(false)} className="space-y-3"> - <input type="hidden" name="friendlyId" value={queue.id} /> - <div className="space-y-2"> - <label htmlFor="concurrencyLimit" className="text-sm text-text-bright"> - Concurrency limit - </label> - <Input - type="number" - name="concurrencyLimit" - id="concurrencyLimit" - min="0" - max={environmentConcurrencyLimit} - value={concurrencyLimit} - onChange={(e) => setConcurrencyLimit(e.target.value)} - placeholder={currentLimit.toString()} - autoFocus - /> - </div> - - <FormButtons - defaultAction={{ - name: "action", - value: "queue-override", - disabled: isLoading || !concurrencyLimit, - }} - confirmButton={ - <Button - type="submit" - name="action" - value="queue-override" - disabled={isLoading || !concurrencyLimit} - variant="primary/medium" - LeadingIcon={isLoading && <Spinner color="white" />} - shortcut={{ modifiers: ["mod"], key: "enter" }} - > - {isOverridden ? "Update override" : "Override limit"} - </Button> - } - cancelButton={ - <div className="flex items-center justify-between gap-2"> - {isOverridden && ( - <Button - type="submit" - name="action" - value="queue-remove-override" - disabled={isLoading} - variant="danger/medium" - > - Remove override - </Button> - )} - <DialogClose asChild> - <Button type="button" variant="tertiary/medium"> - Cancel - </Button> - </DialogClose> - </div> - } - /> - </Form> - </div> - </DialogContent> - </Dialog> - ); -} - -function EngineVersionUpgradeCallout() { - return ( - <div className="mt-4 flex max-w-lg flex-col gap-4 rounded-sm border border-grid-bright bg-background-bright px-4"> - <div className="flex items-center justify-between gap-2 border-b border-grid-dimmed py-4"> - <h4 className="text-base text-text-bright">New queues table</h4> - <LinkButton - LeadingIcon={BookOpenIcon} - to={docsPath("upgrade-to-v4")} - variant={"docs/small"} - > - Upgrade guide - </LinkButton> - </div> - <div className="space-y-4 pb-4"> - <Paragraph variant="small"> - Upgrade to SDK version 4+ to view the new queues table, and be able to pause and resume - individual queues. - </Paragraph> - <img - src={upgradeForQueuesPath} - alt="Upgrade for queues" - className="rounded-sm border border-grid-dimmed" - /> - </div> - </div> - ); -} - -export function isEnvironmentPauseResumeFormSubmission( - formMethod: string | undefined, - formData: FormData | undefined -) { - if (!formMethod || !formData) { - return false; - } - - return ( - formMethod.toLowerCase() === "post" && - (formData.get("action") === "environment-pause" || - formData.get("action") === "environment-resume") - ); -} - -export function QueueFilters() { - return <SearchInput placeholder="Search queues…" paramName="query" resetParams={["page"]} />; -} - function BurstFactorTooltip({ environment, }: { @@ -1071,7 +2015,7 @@ function BurstFactorTooltip({ }, but you can burst up to ${ environment.burstFactor * environment.concurrencyLimit } when across multiple queues/tasks.`} - contentClassName="max-w-xs" + contentClassName="max-w-[230px]" /> ); } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx new file mode 100644 index 00000000000..8dc0c2c24f4 --- /dev/null +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -0,0 +1,1197 @@ +import { type MetaFunction } from "@remix-run/react"; +import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from "react"; +import type { QueueItem } from "@trigger.dev/core/v3/schemas"; +import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { z } from "zod"; +import { MainCenteredContainer, PageContainer } from "~/components/layout/AppLayout"; +import { MetricsLayout } from "~/components/layout/MetricsLayout"; +import { AnimatedOrgBannerBar } from "~/components/billing/AnimatedOrgBannerBar"; +import { BigNumber } from "~/components/metrics/BigNumber"; +import { Header3 } from "~/components/primitives/Headers"; +import { NavBar, PageTitle } from "~/components/primitives/PageHeader"; +import { Spinner } from "~/components/primitives/Spinner"; +import { buildActivityTimeAxis } from "~/components/primitives/charts/activityTimeAxis"; +import { + Chart, + type ChartConfig, + type ChartState, +} from "~/components/primitives/charts/ChartCompound"; +import { ChartCard } from "~/components/primitives/charts/ChartCard"; +import { ChartSyncProvider } from "~/components/primitives/charts/ChartSyncContext"; +import { useZoomToTimeFilter } from "~/hooks/useZoomToTimeFilter"; +import { + QUEUE_METRIC_COLORS as COLORS, + QUEUE_METRICS_DEFAULT_PERIOD, + QueueMetricChartCard as QueueDetailChartCard, + type QueueMetricIds as Ids, + type QueueMetricTimeRange as TimeRangeParams, + clickhouseTimeToMs, + formatWaitMs, + toNumber, + useQueueMetric, +} from "~/components/queues/QueueMetricCards"; +import { findProjectBySlug } from "~/models/project.server"; +import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; +import { QueueRetrievePresenter } from "~/presenters/v3/QueueRetrievePresenter.server"; +import { + Table, + TableBlankRow, + TableBody, + TableCell, + TableHeader, + TableHeaderCell, + TableRow, +} from "~/components/primitives/Table"; +import { TabButton, TabContainer } from "~/components/primitives/Tabs"; +import { InfoIconTooltip } from "~/components/primitives/Tooltip"; +import { SearchInput } from "~/components/primitives/SearchInput"; +import { engine } from "~/v3/runEngine.server"; +import { TimeFilter } from "~/components/runs/v3/SharedFilters"; +import { useSearchParams } from "~/hooks/useSearchParam"; +import { useInterval } from "~/hooks/useInterval"; +import { PaginationControls } from "~/components/primitives/Pagination"; +import type { + ConcurrencyKeyRow, + ConcurrencyKeysResponse, +} from "~/routes/resources.queues.concurrency-keys"; +import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; +import { canAccessQueueMetricsUi } from "~/v3/canAccessQueueMetricsUi.server"; +import { requireUserId } from "~/services/session.server"; +import { docsPath, EnvironmentParamSchema, v3RunsPath } from "~/utils/pathBuilder"; +import { formatNumberCompact } from "~/utils/numberFormatter"; +import { cn } from "~/utils/cn"; +import { redirectWithErrorMessage } from "~/models/message.server"; +import { handleQueueMutationAction } from "~/models/queueMutation.server"; +import { + QueueOverrideConcurrencyButton, + QueuePauseResumeButton, +} from "~/components/queues/QueueControls"; +import { LinkButton } from "~/components/primitives/Buttons"; +import { RunsIcon } from "~/assets/icons/RunsIcon"; +import { InfoPanel } from "~/components/primitives/InfoPanel"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { InlineCode } from "~/components/code/InlineCode"; +import { ConcurrencyIcon } from "~/assets/icons/ConcurrencyIcon"; +import { BookOpenIcon } from "@heroicons/react/20/solid"; + +export const meta: MetaFunction = () => [{ title: `Queue metrics | Trigger.dev` }]; + +const ParamsSchema = EnvironmentParamSchema.extend({ queueParam: z.string() }); + +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const { organizationSlug, projectParam, envParam, queueParam } = ParamsSchema.parse(params); + + // This whole page is part of the metrics UI; gate it per-org (the list already hides + // the only link to it, this is defense in depth). + if (!(await canAccessQueueMetricsUi({ userId, organizationSlug }))) { + throw new Response(undefined, { status: 404, statusText: "Not found" }); + } + + const url = new URL(request.url); + + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + if (!project) throw new Response(undefined, { status: 404, statusText: "Project not found" }); + + const environment = await findEnvironmentBySlug(project.id, envParam, userId); + if (!environment) + throw new Response(undefined, { status: 404, statusText: "Environment not found" }); + + const retrieve = await new QueueRetrievePresenter().call({ environment, queueInput: queueParam }); + if (!retrieve.success) { + throw new Response(undefined, { status: 404, statusText: "Queue not found" }); + } + + const queue = retrieve.queue; + const fullName = queue.type === "task" ? `task/${queue.name}` : queue.name; + + const [ckBreakdown, oldestQueuedAt] = await Promise.all([ + engine.concurrencyKeyBreakdown(environment, fullName, { limit: CK_LIVE_LIMIT }), + // Enqueue time of the oldest run still waiting in the queue right now (any queue, keyed or + // not); undefined when the queue is empty. Drives the live "Oldest wait" block. + engine.oldestMessageInQueue(environment, fullName), + ]); + + // Charts + CH-derived stats are fetched client-side per card (see QueueDetailChartCard / + // useQueueMetric) so the drill-down renders instantly. The loader only returns the live + // "now" counts + identifiers the client fetches need. + // Link the Queued block to this queue's pending runs (same filter the list uses). + const queuedRunsPath = v3RunsPath( + { slug: organizationSlug }, + { slug: projectParam }, + { slug: envParam }, + { queues: [fullName], statuses: ["PENDING"], period: "30d", rootOnly: false } + ); + + return typedjson({ + queue, + fullName, + queuedRunsPath, + // The override dialog caps at the environment's concurrency limit. + environmentConcurrencyLimit: environment.maximumConcurrencyLimit, + ckBreakdown, + oldestQueuedAt: oldestQueuedAt ?? null, + loadedAt: Date.now(), + backPath: url.pathname.replace(/\/[^/]+$/, ""), + ids: { + organizationId: environment.organizationId, + projectId: environment.projectId, + environmentId: environment.id, + }, + }); +}; + +export const action = async ({ request, params }: ActionFunctionArgs) => { + const userId = await requireUserId(request); + const { organizationSlug, projectParam, envParam, queueParam } = ParamsSchema.parse(params); + + const url = new URL(request.url); + const redirectPath = `/orgs/${organizationSlug}/projects/${projectParam}/env/${envParam}/queues/${queueParam}${url.search}`; + + if (request.method.toLowerCase() !== "post") { + return redirectWithErrorMessage(redirectPath, request, "Wrong method"); + } + + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + if (!project) throw new Response(undefined, { status: 404, statusText: "Project not found" }); + + const environment = await findEnvironmentBySlug(project.id, envParam, userId); + if (!environment) + throw new Response(undefined, { status: 404, statusText: "Environment not found" }); + + if (environment.archivedAt) { + return redirectWithErrorMessage(redirectPath, request, "This branch is archived"); + } + + const formData = await request.formData(); + + // Pause/resume/override actions are shared with the Queues list route; here we redirect back to + // the detail page so the user stays put. + const result = await handleQueueMutationAction({ + request, + environment, + userId, + formData, + redirectPath, + }); + if (result) return result; + + return redirectWithErrorMessage(redirectPath, request, "Something went wrong"); +}; + +const CK_LIVE_LIMIT = 50; + +// Whole-queue oldest wait right now: for keyed queues the per-key breakdown carries the oldest +// enqueue time per key, so the queue's oldest is the max wait across keys; otherwise fall back to +// the queue's oldest message directly. Returns null when nothing is waiting. +function wholeQueueOldestWaitMs( + breakdown: CkBreakdown, + oldestQueuedAt: number | null, + now: number +): number | null { + // Only keys with a live backlog (queued > 0) count — a lingering ckIndex entry whose subqueue + // has drained would otherwise over-report the oldest wait. Matches the worstKeyNow guard below. + const waitingKeys = breakdown.keys.filter((k) => k.queued > 0); + if (waitingKeys.length > 0) { + return waitingKeys.reduce((max, k) => Math.max(max, now - k.oldestEnqueuedAt), 0); + } + return oldestQueuedAt !== null ? Math.max(0, now - oldestQueuedAt) : null; +} + +export default function Page() { + const { + queue, + fullName, + queuedRunsPath, + environmentConcurrencyLimit, + ckBreakdown, + oldestQueuedAt, + loadedAt, + backPath, + ids, + } = useTypedLoaderData<typeof loader>(); + const plan = useCurrentPlan(); + // Queue metrics are retained for 30 days in ClickHouse, so cap the picker there even for + // plans whose query-period limit was raised above it — a longer window would render empty. + const planPeriodDays = plan?.v3Subscription?.plan?.limits?.queryPeriodDays?.number; + const maxPeriodDays = Math.min(planPeriodDays ?? 30, 30); + + const { value, replace } = useSearchParams(); + const timeRange: TimeRangeParams = { + period: value("period") ?? null, + from: value("from") ?? null, + to: value("to") ?? null, + }; + + // The Concurrency keys tab exists only for queues with key activity: live keys in the + // ckIndex, or nonzero CK history in the selected range (one cached scalar query decides). + const { rows: gateRows, showLoading: gateLoading } = useQueueMetric( + `SELECT max(max_ck_backlogged) AS peak_keys, max(max_ck_wait_ms) AS peak_wait\nFROM queue_metrics`, + { ids, timeRange, queueName: fullName } + ); + const gateRow = gateRows[0]; + const hasHistory = gateRow + ? toNumber(gateRow.peak_keys) > 0 || toNumber(gateRow.peak_wait) > 0 + : false; + // Whether this queue has any concurrency-key activity to show (live keys in the ckIndex, or + // nonzero CK history in the range). Both tabs always render; when a queue has no keys the + // Concurrency keys tab shows an empty state instead of blank charts/table. + const hasKeys = ckBreakdown.keys.length > 0 || (!gateLoading && hasHistory); + const view = value("view") === "keys" ? "keys" : "overview"; + const selectedKey = value("key"); + + return ( + <PageContainer> + <NavBar> + <PageTitle title={queue.name} backButton={{ to: backPath, text: "Queues" }} /> + </NavBar> + {/* Paused-queue banner — mirrors the environment-paused banner (OrgBanner) at the top of + the page when this individual queue is paused. */} + <AnimatedOrgBannerBar show={queue.paused} variant="warning"> + {`"${queue.name}" queue paused. No new runs will be dequeued and executed.`} + </AnimatedOrgBannerBar> + <MetricsLayout.Root> + {/* Filters — search (concurrency keys) + time filter in one left cluster, above + everything, like the Queues list. The time filter scopes the tab charts; search filters + the keys table. The bar is pinned by the layout while the page scrolls. */} + <MetricsLayout.Filters className="pl-1.5 pr-2"> + <div className="translate-y-px self-end pl-2"> + <TabContainer> + <TabButton + isActive={view === "overview"} + layoutId="queue-detail-view" + onClick={() => replace({ view: undefined, key: undefined })} + > + Overview + </TabButton> + <TabButton + isActive={view === "keys"} + layoutId="queue-detail-view" + onClick={() => replace({ view: "keys" })} + > + Concurrency keys + </TabButton> + </TabContainer> + </div> + <div className="flex items-center gap-1.5"> + {view === "keys" && hasKeys ? ( + <SearchInput + placeholder="Search keys…" + paramName="query" + resetParams={["key", "page"]} + /> + ) : null} + <TimeFilter + defaultPeriod={QUEUE_METRICS_DEFAULT_PERIOD} + labelName="Period" + maxPeriodDays={maxPeriodDays} + shortcut={{ key: "d" }} + /> + <QueueOverrideConcurrencyButton + queue={queue} + environmentConcurrencyLimit={environmentConcurrencyLimit} + trigger="button" + /> + <QueuePauseResumeButton + queue={{ id: queue.id, name: queue.name, paused: queue.paused }} + variant="secondary/small" + withQueueName + /> + </div> + </MetricsLayout.Filters> + + {/* Live "right now" state of the whole queue — independent of the time filter above. + QueueStats renders the stat-tile grid slot (see MetricsLayout.Grid inside it). */} + <QueueStats + queue={queue} + environmentConcurrencyLimit={environmentConcurrencyLimit} + queuedRunsPath={queuedRunsPath} + oldestWaitMs={wholeQueueOldestWaitMs(ckBreakdown, oldestQueuedAt, loadedAt)} + ids={ids} + timeRange={timeRange} + queueName={fullName} + /> + + {/* Tabs + charts share the padded (inset) column. Both tabs always render; the keys tab + shows an empty state when the queue has no concurrency keys. */} + <MetricsLayout.Content inset> + {view === "keys" ? ( + hasKeys ? ( + <ConcurrencyKeyCharts + breakdown={ckBreakdown} + loadedAt={loadedAt} + ids={ids} + timeRange={timeRange} + queueName={fullName} + /> + ) : ( + <ConcurrencyKeysBlankState /> + ) + ) : ( + <OverviewCharts ids={ids} timeRange={timeRange} queueName={fullName} /> + )} + </MetricsLayout.Content> + + {/* The per-key table is full-bleed (no inset), matching the Queues list table, so it spans + edge to edge. The drill-down that opens under it is charts, so it stays in the padded + column. */} + {view === "keys" && hasKeys ? ( + <> + <MetricsLayout.Content> + <KeyStatsTable ids={ids} timeRange={timeRange} queueName={fullName} /> + </MetricsLayout.Content> + {selectedKey ? ( + <MetricsLayout.Content inset> + <KeyDrilldown + keyName={selectedKey} + ids={ids} + timeRange={timeRange} + queueName={fullName} + /> + </MetricsLayout.Content> + ) : null} + </> + ) : null} + </MetricsLayout.Root> + </PageContainer> + ); +} + +// Inline colour swatch for tooltip copy — matches the chart legend swatch (rounded-[2px]) and is +// nudged up 1px so it sits on the text baseline. +function ColorSwatch({ color }: { color: string }) { + return ( + <span + className="mx-0.5 inline-block size-2.5 -translate-y-px rounded-[2px] align-middle" + style={{ backgroundColor: color }} + /> + ); +} + +function OverviewCharts({ + ids, + timeRange, + queueName, +}: { + ids: Ids; + timeRange: TimeRangeParams; + queueName: string; +}) { + const zoomToTimeFilter = useZoomToTimeFilter(); + return ( + <ChartSyncProvider onZoom={zoomToTimeFilter}> + <div className="grid grid-cols-1 gap-3 sm:grid-cols-2"> + <QueueDetailChartCard + title="Concurrency" + info={ + <> + How many runs are executing at once (<ColorSwatch color={COLORS.running} />) versus + the queue's limit (<ColorSwatch color={COLORS.limit} /> + ). Turns <ColorSwatch color="var(--color-warning)" /> color when it reaches the limit. + </> + } + showLegend + className="aspect-[2/1]" + query={`SELECT timeBucket() AS t, max(max_running) AS running, max(max_limit) AS limit\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + fillGaps + ids={ids} + timeRange={timeRange} + queueName={queueName} + series={[ + // Limit first so the grey reference draws underneath; Running (purple) sits on top. + { key: "limit", label: "Limit", color: COLORS.limit }, + { key: "running", label: "Running", color: COLORS.running }, + ]} + // Recolour Running above the limit line with a gradient split, so it's orange only where + // it's actually over the limit — not on the way up. The threshold reads off the (roughly + // constant) limit series. + thresholdStroke={{ + series: "running", + valueFromSeries: "limit", + aboveColor: "var(--color-warning)", + }} + // The limit is a config value emitted only while the queue is active; back-fill its + // leading zeros so the reference line doesn't start with a false 0→limit step. + carryBackfill={["limit"]} + /> + <QueueDetailChartCard + title="Queue depth" + info="How many runs are waiting in this queue over time." + className="aspect-[2/1]" + query={`SELECT timeBucket() AS t, max(max_queued) AS queued\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + fillGaps + ids={ids} + timeRange={timeRange} + queueName={queueName} + series={[{ key: "queued", label: "Queued", color: COLORS.queued }]} + /> + <QueueDetailChartCard + title="Throughput" + info={ + <> + Runs arriving (<ColorSwatch color={COLORS.limit} /> Enqueued) versus starting ( + <ColorSwatch color={COLORS.running} /> Started). Turns{" "} + <ColorSwatch color="var(--color-warning)" /> color when Started falls behind. + </> + } + showLegend + extraLegend={[{ color: "var(--color-warning)", label: "Falling behind" }]} + className="aspect-[2/1]" + query={`SELECT timeBucket() AS t,\n deltaSumTimestampMerge(enqueue_delta) AS enqueued,\n deltaSumTimestampMerge(started_delta) AS started\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + fillGaps + ids={ids} + timeRange={timeRange} + queueName={queueName} + series={[ + // Enqueued is the neutral grey reference (same grey as the Limit line on Concurrency); + // Started is the accent — purple while keeping up, warning where it drops below Enqueued. + { key: "enqueued", label: "Enqueued", color: COLORS.limit }, + { key: "started", label: "Started", color: COLORS.running }, + ]} + warningOverlay={{ series: "started", below: "enqueued" }} + /> + <QueueDetailChartCard + title="Scheduling delay" + info="How long runs wait before they start." + showLegend + className="aspect-[2/1]" + query={`SELECT timeBucket() AS t,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[1]) AS p50,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[4]) AS p99\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + fillGaps + ids={ids} + timeRange={timeRange} + queueName={queueName} + valueFormat={formatWaitMs} + series={[ + { key: "p50", label: "p50", color: COLORS.p50 }, + { key: "p95", label: "p95", color: COLORS.p95 }, + { key: "p99", label: "p99", color: COLORS.p99 }, + ]} + /> + <QueueDetailChartCard + title="Throttled" + info={ + <> + How often runs were held back by a limit (<ColorSwatch color={COLORS.throttled} />{" "} + color). + </> + } + className="aspect-[2/1] sm:col-span-2 sm:aspect-[4/1]" + query={`SELECT timeBucket() AS t, sum(throttled_count) AS throttled\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + fillGaps + ids={ids} + timeRange={timeRange} + queueName={queueName} + series={[{ key: "throttled", label: "Throttled", color: COLORS.throttled }]} + /> + </div> + </ChartSyncProvider> + ); +} + +type CkBreakdown = { + totalBackloggedKeys: number; + keys: Array<{ + concurrencyKey: string; + queued: number; + running: number; + oldestEnqueuedAt: number; + }>; +}; + +// Standard empty state for a queue that has no concurrency keys (no live keys and no CK history). +// Small, centered panel — same pattern as the runs page's "Create your first task" state. +function ConcurrencyKeysBlankState() { + return ( + <MainCenteredContainer className="max-w-md"> + <InfoPanel + icon={ConcurrencyIcon} + iconClassName="text-text-dimmed" + panelClassName="max-w-full" + title="No concurrency keys configured" + accessory={ + <LinkButton + to={docsPath("/queue-concurrency")} + variant="docs/small" + LeadingIcon={BookOpenIcon} + > + Concurrency docs + </LinkButton> + } + > + <Paragraph variant="small"> + This queue doesn't use concurrency keys. Add <InlineCode>concurrencyKey</InlineCode> to + your task to shard the queue per tenant/user. + </Paragraph> + </InfoPanel> + </MainCenteredContainer> + ); +} + +function ConcurrencyKeyCharts({ + breakdown, + loadedAt, + ids, + timeRange, + queueName, +}: { + breakdown: CkBreakdown; + loadedAt: number; + ids: Ids; + timeRange: TimeRangeParams; + queueName: string; +}) { + const zoomToTimeFilter = useZoomToTimeFilter(); + const { value, replace } = useSearchParams(); + // Same key search as the table: narrows the per-key charts too. + const keyFilter = value("query")?.trim().toLowerCase() || undefined; + + // The live most-starved key: among keys with a live backlog, the one whose oldest waiting run + // has been waiting longest right now. Names the culprit on the "Worst key wait" card so the chart + // (which shows how bad, not who) points at a tenant you can click through to. + const worstKeyNow = useMemo(() => { + let worst: { key: string; waitMs: number } | null = null; + for (const k of breakdown.keys) { + if (k.queued <= 0) continue; + const waitMs = Math.max(0, loadedAt - k.oldestEnqueuedAt); + if (!worst || waitMs > worst.waitMs) worst = { key: k.concurrencyKey, waitMs }; + } + return worst; + }, [breakdown, loadedAt]); + + return ( + /* Per-key breakdown: which keys hold the backlog / do the work. */ + <ChartSyncProvider onZoom={zoomToTimeFilter}> + <div className="grid grid-cols-1 gap-3 sm:grid-cols-2"> + <GroupedKeyChartCard + title="Waiting runs by key" + info="Runs waiting per key (top 8)." + className="aspect-[2/1]" + rankExpr="max(max_queued)" + seriesExpr="max(max_queued)" + fillGaps + keyFilter={keyFilter} + ids={ids} + timeRange={timeRange} + queueName={queueName} + /> + <GroupedKeyChartCard + title="Throughput by key" + info="Runs started per key (top 8)." + className="aspect-[2/1]" + rankExpr="deltaSumTimestampMerge(started_delta)" + seriesExpr="deltaSumTimestampMerge(started_delta)" + keyFilter={keyFilter} + ids={ids} + timeRange={timeRange} + queueName={queueName} + /> + {/* Whole-queue health across keys (single series, queues-purple). */} + <QueueDetailChartCard + title="Keys with backlog" + info="Keys with runs waiting at once." + className="aspect-[2/1]" + query={`SELECT timeBucket() AS t, max(max_ck_backlogged) AS keys\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + fillGaps + ids={ids} + timeRange={timeRange} + queueName={queueName} + series={[{ key: "keys", label: "Keys", color: COLORS.running }]} + /> + <QueueDetailChartCard + title="Worst key wait" + info="Longest wait across all keys." + titleAccessory={ + worstKeyNow ? ( + <button + type="button" + onClick={() => replace({ key: worstKeyNow.key })} + className="cursor-pointer text-xs font-normal tabular-nums text-text-dimmed transition-colors hover:text-text-bright" + > + {worstKeyNow.key} · {formatWaitMs(worstKeyNow.waitMs)} now + </button> + ) : null + } + className="aspect-[2/1]" + query={`SELECT timeBucket() AS t, max(max_ck_wait_ms) AS wait\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + fillGaps + ids={ids} + timeRange={timeRange} + queueName={queueName} + valueFormat={formatWaitMs} + series={[{ key: "wait", label: "Max wait", color: COLORS.running }]} + /> + </div> + </ChartSyncProvider> + ); +} + +// TRQL string literal escape (standard SQL doubling). +function trqlString(value: string): string { + return value.replace(/'/g, "''"); +} + +const KEY_SERIES_COLORS = [ + "#34D399", + "#6366F1", + "#F59E0B", + "#22D3EE", + "#A78BFA", + "#EF4444", + "#F472B6", + "#84CC16", +]; + +// A card title with an info "i" tooltip, matching the Queues header charts. Used by the grouped +// per-key charts (the plain single-series cards get the same treatment via QueueMetricChartCard). +function chartTitleWithInfo(title: string, info: string) { + return ( + <span className="flex items-center gap-1.5"> + {title} + <InfoIconTooltip content={info} contentClassName="max-w-[230px]" /> + </span> + ); +} + +type GroupedKeyChartProps = { + title: string; + info: string; + className?: string; + /** Aggregate expression ranking keys over the whole range (top 8 charted). */ + rankExpr: string; + /** Aggregate expression charted per (bucket, key). */ + seriesExpr: string; + fillGaps?: boolean; + valueFormat?: (value: number) => string; + /** Search substring (from the page's key search) — narrows the charted keys, like the table. */ + keyFilter?: string; + ids: Ids; + timeRange: TimeRangeParams; + queueName: string; +}; + +// Two-step top-N: rank keys over the range, then chart those keys as grouped series +// (the per-key table is activity-bound, so ranking is a cheap scan). Rank wider than we chart so a +// search can match keys outside the top 8; then filter by the search and keep the top 8 of those. +function GroupedKeyChartCard(props: GroupedKeyChartProps) { + const { rows, showLoading, failed } = useQueueMetric( + `SELECT concurrency_key, ${props.rankExpr} AS peak\nFROM queue_metrics_by_key\nGROUP BY concurrency_key\nORDER BY peak DESC\nLIMIT 50`, + { ids: props.ids, timeRange: props.timeRange, queueName: props.queueName } + ); + const keyFilter = props.keyFilter; + const keys = useMemo(() => { + let names = rows.filter((r) => toNumber(r.peak) > 0).map((r) => String(r.concurrency_key)); + if (keyFilter) names = names.filter((n) => n.toLowerCase().includes(keyFilter)); + return names.slice(0, 8); + }, [rows, keyFilter]); + + if (showLoading || failed || keys.length === 0) return null; + return <GroupedKeySeries keys={keys} {...props} />; +} + +function GroupedKeySeries({ + keys, + title, + info, + className, + seriesExpr, + fillGaps, + valueFormat, + ids, + timeRange, + queueName, +}: GroupedKeyChartProps & { keys: string[] }) { + const inList = keys.map((k) => `'${trqlString(k)}'`).join(", "); + const { rows, showLoading, failed } = useQueueMetric( + `SELECT timeBucket() AS t, concurrency_key, ${seriesExpr} AS v\nFROM queue_metrics_by_key\nWHERE concurrency_key IN (${inList})\nGROUP BY t, concurrency_key\nORDER BY t`, + { ids, timeRange, queueName, fillGaps } + ); + + const data = useMemo(() => { + const buckets = new Map<number, { bucket: number } & Record<string, number>>(); + for (const r of rows) { + const bucket = clickhouseTimeToMs(r.t); + if (!Number.isFinite(bucket)) continue; + let point = buckets.get(bucket); + if (!point) { + point = { bucket } as { bucket: number } & Record<string, number>; + buckets.set(bucket, point); + } + point[String(r.concurrency_key)] = toNumber(r.v); + } + return [...buckets.values()].sort((a, b) => a.bucket - b.bucket); + }, [rows]); + + const chartConfig = useMemo(() => { + const cfg: ChartConfig = {}; + keys.forEach((k, i) => { + cfg[k] = { label: k, color: KEY_SERIES_COLORS[i % KEY_SERIES_COLORS.length]! }; + }); + return cfg; + }, [keys]); + + const { tickFormatter, tooltipLabelFormatter } = useMemo( + () => buildActivityTimeAxis(data), + [data] + ); + const state: ChartState = showLoading ? "loading" : failed ? "invalid" : undefined; + + return ( + <div className={className ?? "h-64"}> + <ChartCard title={chartTitleWithInfo(title, info)}> + <Chart.Root + config={chartConfig} + data={data} + dataKey="bucket" + series={keys} + state={state} + fillContainer + > + <Chart.Line + lineType="monotone" + xAxisProps={{ tickFormatter }} + yAxisProps={valueFormat ? { tickFormatter: (v: number) => valueFormat(v) } : undefined} + tooltipLabelFormatter={tooltipLabelFormatter} + tooltipValueFormatter={valueFormat} + /> + </Chart.Root> + </ChartCard> + </div> + ); +} + +// One page of the paginated per-key table. The ClickHouse tier is the authority (ranked by peak +// backlog over the window, with the total on every row so page + count are a single scan); each +// page's keys are enriched with live "now" counts from Redis server-side. Fetched per page rather +// than capped at 50, so high-cardinality queues (tens of thousands of keys) page through instead +// of silently truncating. See resources.queues.concurrency-keys. +function useConcurrencyKeys(opts: { + ids: Ids; + timeRange: TimeRangeParams; + queueName: string; + search: string; + page: number; +}) { + const { ids, timeRange, queueName, search, page } = opts; + const [data, setData] = useState<ConcurrencyKeysResponse | null>(null); + const [isLoading, setIsLoading] = useState(true); + const abortRef = useRef<AbortController | null>(null); + + const body = useMemo( + () => + JSON.stringify({ + organizationId: ids.organizationId, + projectId: ids.projectId, + environmentId: ids.environmentId, + queueName, + period: timeRange.period, + from: timeRange.from, + to: timeRange.to, + search, + page, + }), + [ + ids.organizationId, + ids.projectId, + ids.environmentId, + queueName, + timeRange.period, + timeRange.from, + timeRange.to, + search, + page, + ] + ); + + const load = useCallback(() => { + abortRef.current?.abort(); + const controller = new AbortController(); + abortRef.current = controller; + setIsLoading(true); + fetch("/resources/queues/concurrency-keys", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body, + signal: controller.signal, + }) + .then((res) => res.json() as Promise<ConcurrencyKeysResponse>) + .then((res) => { + if (controller.signal.aborted) return; + setData(res); + setIsLoading(false); + }) + .catch((error) => { + if (error instanceof DOMException && error.name === "AbortError") return; + if (!controller.signal.aborted) { + setData({ success: false, error: error?.message ?? "Network error" }); + setIsLoading(false); + } + }); + }, [body]); + + useEffect(() => { + load(); + return () => abortRef.current?.abort(); + }, [load]); + + // Keep the live "now" counts fresh without a manual reload. + useInterval({ + interval: 30_000, + onLoad: false, + onFocus: true, + pauseWhenHidden: true, + callback: load, + }); + + return { data, isLoading }; +} + +// Paginated per-key table: which keys hold the backlog / do the work. Clicking a key pins the +// drill-down charts via the `key` search param. +function KeyStatsTable({ + ids, + timeRange, + queueName, +}: { + ids: Ids; + timeRange: TimeRangeParams; + queueName: string; +}) { + const { value, replace, del } = useSearchParams(); + const selectedKey = value("key"); + const search = value("query")?.trim() ?? ""; + const page = Math.max(1, Number(value("page")) || 1); + + const { data, isLoading } = useConcurrencyKeys({ ids, timeRange, queueName, search, page }); + + const rows: ConcurrencyKeyRow[] = data?.success ? data.rows : []; + const total = data?.success ? data.total : 0; + const perPage = data?.success ? data.perPage : 25; + const totalPages = Math.max(1, Math.ceil(total / perPage)); + // Only show a skeleton before the first response; keep prior rows visible while revalidating. + const showLoading = isLoading && !data; + + // Recover from a page past the end: narrowing the time range (or a stale bookmarked URL) can + // leave `page` beyond the result set, which comes back empty with the pagination control — shown + // only when there's >1 page — hidden, stranding the reader. Once a response settles empty on a + // page > 1, snap back to page 1 (whose data is fetched fresh) so there's always a way out. + useEffect(() => { + if (!isLoading && data?.success && data.rows.length === 0 && page > 1) { + del("page"); + } + }, [isLoading, data, page, del]); + + return ( + <div className="flex flex-col"> + {/* Title bar above the table, shown only when there's more than one page: the section title + on the left, prev/next pagination on the right. Hidden entirely for a single page. */} + {totalPages > 1 ? ( + <div className="flex items-center justify-between border-t px-3 py-2"> + <Header3>Concurrency keys</Header3> + <PaginationControls currentPage={page} totalPages={totalPages} showPageNumbers={false} /> + </div> + ) : null} + {/* Full-bleed, edge-to-edge like the Queues list table: a top border, no rounded side box. */} + <Table containerClassName="border-t"> + <TableHeader> + <TableRow> + <TableHeaderCell>Key</TableHeaderCell> + <TableHeaderCell alignment="right">Queued now</TableHeaderCell> + <TableHeaderCell alignment="right">Running now</TableHeaderCell> + <TableHeaderCell alignment="right">Oldest wait</TableHeaderCell> + <TableHeaderCell alignment="right">Started</TableHeaderCell> + <TableHeaderCell alignment="right">Peak backlog</TableHeaderCell> + <TableHeaderCell alignment="right">Mean delay</TableHeaderCell> + </TableRow> + </TableHeader> + <TableBody> + {showLoading ? ( + <TableBlankRow colSpan={7} className="text-text-dimmed"> + Loading… + </TableBlankRow> + ) : rows.length === 0 ? ( + <TableBlankRow colSpan={7} className="text-text-dimmed"> + {search ? `No keys match “${search}”` : "No concurrency keys"} + </TableBlankRow> + ) : ( + rows.map((row) => ( + <TableRow + key={row.key} + isSelected={selectedKey === row.key} + className="cursor-pointer" + onClick={() => (selectedKey === row.key ? del("key") : replace({ key: row.key }))} + > + <TableCell>{row.key}</TableCell> + <TableCell alignment="right">{row.queued.toLocaleString()}</TableCell> + <TableCell alignment="right">{row.running.toLocaleString()}</TableCell> + <TableCell alignment="right"> + {row.oldestWaitMs === null ? "–" : formatWaitMs(row.oldestWaitMs)} + </TableCell> + <TableCell alignment="right">{row.started.toLocaleString()}</TableCell> + <TableCell alignment="right">{row.peakBacklog.toLocaleString()}</TableCell> + <TableCell alignment="right"> + {row.meanWaitMs > 0 ? formatWaitMs(row.meanWaitMs) : "–"} + </TableCell> + </TableRow> + )) + )} + </TableBody> + </Table> + </div> + ); +} + +function KeyDrilldown({ + keyName, + ids, + timeRange, + queueName, +}: { + keyName: string; + ids: Ids; + timeRange: TimeRangeParams; + queueName: string; +}) { + const pin = `concurrency_key = '${trqlString(keyName)}'`; + const zoomToTimeFilter = useZoomToTimeFilter(); + return ( + <ChartSyncProvider onZoom={zoomToTimeFilter}> + <div className="grid grid-cols-1 gap-3 sm:grid-cols-2"> + <QueueDetailChartCard + title={`Key ${keyName}: backlog and running`} + info={ + <> + This key: waiting (Queued, <ColorSwatch color={COLORS.queued} /> color) vs running ( + <ColorSwatch color={COLORS.limit} /> color). + </> + } + className="aspect-[2/1]" + query={`SELECT timeBucket() AS t, max(max_queued) AS queued, max(max_running) AS running\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} + fillGaps + ids={ids} + timeRange={timeRange} + queueName={queueName} + series={[ + // Running is the grey reference underneath; Queued (the backlog we care about) is the purple accent on top. + { key: "running", label: "Running", color: COLORS.limit }, + { key: "queued", label: "Queued", color: COLORS.running }, + ]} + /> + <QueueDetailChartCard + title={`Key ${keyName}: throughput`} + className="aspect-[2/1]" + query={`SELECT timeBucket() AS t, deltaSumTimestampMerge(started_delta) AS started\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} + ids={ids} + timeRange={timeRange} + queueName={queueName} + series={[{ key: "started", label: "Started", color: COLORS.running }]} + /> + <QueueDetailChartCard + title={`Key ${keyName}: mean scheduling delay`} + className="aspect-[2/1]" + query={`SELECT timeBucket() AS t, if(sum(wait_ms_count) > 0, round(sum(wait_ms_sum) / sum(wait_ms_count)), 0) AS wait\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} + ids={ids} + timeRange={timeRange} + queueName={queueName} + valueFormat={formatWaitMs} + series={[{ key: "wait", label: "Mean delay", color: COLORS.running }]} + /> + </div> + </ChartSyncProvider> + ); +} + +// Live "right now" snapshot of the whole queue: a hybrid feed. The loader (Redis/PG) supplies the +// first paint so these blocks render instantly; after that a tiny 15s ClickHouse poll keeps them +// fresh, always reading the newest gauge row and falling back to the loader values until the first +// poll lands (so we never flash 0). These blocks never change with the filter. Period trends +// (backlog, throughput, delay over time) live in the charts below. +// Oldest-wait threshold for the warning tint: the head of the queue sitting unstarted this long +// signals the queue is stuck, not just busy. +const OLDEST_WAIT_WARNING_MS = 5 * 60_000; + +// How recent the newest ClickHouse gauge bucket must be to drive the live blocks. Above the 10s +// bucket + pipeline lag; past it we treat the queue as idle and fall back to the loader value. +const LIVE_GAUGE_FRESH_MS = 90_000; + +function QueueStats({ + queue, + environmentConcurrencyLimit, + queuedRunsPath, + oldestWaitMs, + ids, + timeRange, + queueName, +}: { + // Carries the percent override source-of-truth (not part of the shared QueueItem contract) so the + // override dialog reopens in percent mode for percent-based overrides. + queue: QueueItem & { concurrencyLimitOverridePercent: number | null }; + environmentConcurrencyLimit: number; + queuedRunsPath: string; + oldestWaitMs: number | null; + ids: Ids; + timeRange: TimeRangeParams; + queueName: string; +}) { + // Live "now" is empty for a queue that just drained, so add the window peak/worst as a dimmed + // caption (styled like "bursts up to 50" on the Queues list) — a quiet queue still shows how + // backed up it got recently. "worst_wait" is the window's worst head-of-line wait (max of the + // oldest still-waiting run's age), the same metric as the live "Oldest wait" headline — not the + // scheduling-delay percentiles of runs that already started. Only concurrency-keyed queues record + // this (max_ck_wait_ms), so it's 0/absent for non-keyed queues. + const { rows } = useQueueMetric( + `SELECT max(max_queued) AS peak_queued,\n max(max_ck_wait_ms) AS worst_wait\nFROM queue_metrics`, + { ids, timeRange, queueName } + ); + const peakQueued = rows[0] ? toNumber(rows[0].peak_queued) : 0; + const worstWaitMs = rows[0] ? toNumber(rows[0].worst_wait) : 0; + + // Latest gauges from ClickHouse, polled every 15s so the live blocks keep ticking after first + // paint. Read the newest bucket (largest t); until the first poll lands liveRows is empty and the + // *Live values stay null, so the blocks show the loader values instead of flashing 0. + const { rows: liveRows } = useQueueMetric( + `SELECT timeBucket() AS t, max(max_running) AS running, max(max_queued) AS queued, max(max_limit) AS q_limit, max(max_ck_wait_ms) AS ck_wait FROM queue_metrics GROUP BY t ORDER BY t`, + { + ids, + timeRange: { period: "15m", from: null, to: null }, + defaultPeriod: "15m", + queueName, + refreshIntervalMs: 15_000, + } + ); + // Gauges are only emitted while the queue is active, so a drained queue's newest bucket is a past + // one holding its last non-zero reading. Trust the CH gauge only when its newest bucket is recent + // (covers the 10s bucket + pipeline lag); once it ages out we fall back to the loader's live + // Redis/PG value instead of lingering on a stale count. + const latest = liveRows.length > 0 ? liveRows[liveRows.length - 1] : undefined; + const latestBucketMs = latest ? clickhouseTimeToMs(latest.t) : NaN; + const liveFresh = + Number.isFinite(latestBucketMs) && Date.now() - latestBucketMs < LIVE_GAUGE_FRESH_MS; + const fresh = latest && liveFresh ? latest : undefined; + const runningLive = fresh ? toNumber(fresh.running) : null; + const queuedLive = fresh ? toNumber(fresh.queued) : null; + const limitLive = fresh ? toNumber(fresh.q_limit) : null; + const ckWaitLive = fresh ? toNumber(fresh.ck_wait) : null; + + // Prefer CH once it has landed; loader values before that. + const runningDisplay = runningLive ?? queue.running; + const queuedDisplay = queuedLive ?? queue.queued; + // Limit is queue config, not a live signal: keep the loader's value. Only if the loader had none + // do we fall back to the CH gauge for display. + const limitDisplay = queue.concurrencyLimit ?? (limitLive || null); + // Keyed queues report head-of-line wait via CH (max_ck_wait_ms); use it as the live headline when + // present. Non-keyed queues have no CH signal, so they stay on the loader value. + const oldestWaitDisplayMs = ckWaitLive !== null && ckWaitLive > 0 ? ckWaitLive : oldestWaitMs; + + return ( + <MetricsLayout.Grid> + <ConcurrencyBlock running={runningDisplay} limit={limitDisplay} paused={queue.paused} /> + <BigNumber + title="Queued" + value={queuedDisplay} + valueClassName="tabular-nums" + animate + suffix={peakQueued > 0 ? `peak ${formatNumberCompact(peakQueued)}` : undefined} + suffixClassName="text-text-dimmed" + accessory={ + <span className="opacity-0 transition-opacity group-focus-within:opacity-100 group-hover:opacity-100"> + <LinkButton + variant="minimal/small" + className="aspect-square px-1!" + LeadingIcon={RunsIcon} + leadingIconClassName="text-text-dimmed group-hover/button:text-text-bright" + to={queuedRunsPath} + tooltip="View queued runs" + /> + </span> + } + /> + <BigNumber + title="Oldest wait" + formattedValue={ + oldestWaitDisplayMs !== null && oldestWaitDisplayMs > 0 + ? formatWaitMs(oldestWaitDisplayMs) + : "0" + } + valueClassName={cn( + "tabular-nums", + oldestWaitDisplayMs !== null && + oldestWaitDisplayMs >= OLDEST_WAIT_WARNING_MS && + "text-warning" + )} + suffix={worstWaitMs > 0 ? `worst ${formatWaitMs(worstWaitMs)}` : undefined} + suffixClassName="text-text-dimmed" + /> + </MetricsLayout.Grid> + ); +} + +/** Live concurrency as a single block: running vs limit with a utilization bar, so "how close to + * the ceiling" reads at a glance instead of two separate numbers. Warning-tinted at/over the limit. */ +function ConcurrencyBlock({ + running, + limit, + paused = false, + loading, + accessory, +}: { + running: number; + limit: number | null; + paused?: boolean; + loading?: boolean; + accessory?: ReactNode; +}) { + const atLimit = limit !== null && limit > 0 && running >= limit; + const pct = limit && limit > 0 ? Math.min(100, Math.round((running / limit) * 100)) : 0; + return ( + <div className="flex flex-col justify-between gap-4 rounded-lg border border-grid-bright bg-background-bright p-4"> + <div className="flex items-center justify-between gap-2"> + <div className="flex items-center gap-2"> + <Header3 className="leading-6">Concurrency</Header3> + {paused ? <span className="text-xs text-warning">paused</span> : null} + </div> + {accessory ? <div className="shrink-0">{accessory}</div> : null} + </div> + {loading ? ( + <Spinner className="size-6" /> + ) : ( + <div className="flex flex-col gap-2"> + <div className="flex flex-wrap items-baseline gap-2"> + <span + className={cn( + "text-[3.75rem] font-normal leading-none tabular-nums", + atLimit ? "text-warning" : "text-text-bright" + )} + > + {running.toLocaleString()} + </span> + <span className="text-xl tabular-nums text-text-dimmed"> + / {limit !== null ? limit.toLocaleString() : "∞"} + </span> + {limit !== null && limit > 0 && ( + <span + className={cn( + "text-xs tabular-nums", + atLimit ? "text-warning" : "text-text-dimmed" + )} + > + {/* Separator so the limit and the percentage don't read as one number + (e.g. "/ 25" + "44%" mashing into "2544%"). */} + <span className="mr-1 text-text-dimmed">·</span> + {pct}% of limit + </span> + )} + </div> + {limit !== null && limit > 0 && ( + <div className="h-1.5 w-full overflow-hidden rounded-full bg-charcoal-750"> + <div + className={cn("h-full rounded-full", atLimit ? "bg-warning" : "bg-queues")} + style={{ width: `${pct}%` }} + /> + </div> + )} + </div> + )} + </div> + ); +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx index 398c6135531..6d81de239fe 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx @@ -40,6 +40,12 @@ import { } from "~/components/primitives/Resizable"; import { Sheet, SheetContent } from "~/components/primitives/SheetV3"; import { Spinner } from "~/components/primitives/Spinner"; +import { TextLink } from "~/components/primitives/TextLink"; +import { + QueueSidebarStats, + type QueueLiveCounts, + type QueueMetricIds, +} from "~/components/queues/QueueMetricCards"; import { Table, TableBlankRow, @@ -83,12 +89,15 @@ import { v3EditSchedulePath, v3EnvironmentPath, v3NewSchedulePath, + v3QueuePath, v3RunsPath, v3SchedulePath, v3SchedulesAddOnPath, v3TestTaskPath, } from "~/utils/pathBuilder"; import { parseFiniteInt } from "~/utils/searchParams"; +import { canAccessQueueMetricsUi } from "~/v3/canAccessQueueMetricsUi.server"; +import { engine } from "~/v3/runEngine.server"; import type { loader as scheduleDetailLoader } from "../_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.$scheduleParam/route"; import type { loader as scheduleEditLoader } from "../_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.edit.$scheduleParam/route"; import type { loader as scheduleNewLoader } from "../_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.new/route"; @@ -141,6 +150,28 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { if (!task) throw new Response("Scheduled task not found", { status: 404 }); + // Live queue counts for the sidebar Queue property (flag on only; the property itself + // is not rendered without them, so flag off = no extra reads and no UI change). + let queueMetrics: { live: QueueLiveCounts; ids: QueueMetricIds } | null = null; + if (task.queue && (await canAccessQueueMetricsUi({ userId, organizationSlug }))) { + const queueName = task.queue.name; + const [lengths, concurrency] = await Promise.all([ + engine.lengthOfQueues(environment, [queueName]), + engine.currentConcurrencyOfQueues(environment, [queueName]), + ]); + queueMetrics = { + live: { + queued: lengths?.[queueName] ?? 0, + running: concurrency?.[queueName] ?? 0, + }, + ids: { + organizationId: project.organizationId, + projectId: project.id, + environmentId: environment.id, + }, + }; + } + const time = timeFilterFromTo({ period, from, to, defaultPeriod: "7d" }); const activity = taskPresenter @@ -190,11 +221,13 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { activity, scheduleList, runList, + queueMetrics, }); }; export default function Page() { - const { task, activity, scheduleList, runList } = useTypedLoaderData<typeof loader>(); + const { task, activity, scheduleList, runList, queueMetrics } = + useTypedLoaderData<typeof loader>(); const zoomToTimeFilter = useZoomToTimeFilter(); const organization = useOrganization(); const project = useProject(); @@ -204,6 +237,9 @@ export default function Page() { const testPath = v3TestTaskPath(organization, project, environment, { taskIdentifier: task.slug, }); + const queuePath = task.queue + ? v3QueuePath(organization, project, environment, { friendlyId: task.queue.friendlyId }) + : undefined; const filters: TaskRunListSearchFilters = useMemo(() => ({ tasks: [task.slug] }), [task.slug]); @@ -378,6 +414,8 @@ export default function Page() { testPath={testPath} scheduleList={scheduleList} onSelectSchedule={openSchedule} + queuePath={queuePath} + queueMetrics={queueMetrics} /> </ResizablePanel> </ResizablePanelGroup> @@ -724,10 +762,15 @@ function ScheduledTaskDetailSidebar({ testPath, scheduleList, onSelectSchedule, -}: { task: TaskDetail; testPath: string; onSelectSchedule: (friendlyId: string) => void } & Pick< - LoaderData, - "scheduleList" ->) { + queuePath, + queueMetrics, +}: { + task: TaskDetail; + testPath: string; + onSelectSchedule: (friendlyId: string) => void; + queuePath: string | undefined; + queueMetrics: { live: QueueLiveCounts; ids: QueueMetricIds } | null; +} & Pick<LoaderData, "scheduleList">) { const sortedSchedules = useMemo(() => { if (!scheduleList) return []; // DECLARATIVE first; createdAt-desc within each type (stable sort). @@ -859,6 +902,26 @@ function ScheduledTaskDetailSidebar({ )} </Property.Value> </Property.Item> + {queueMetrics && task.queue && queuePath ? ( + <Property.Item> + <Property.Label>Queue</Property.Label> + <Property.Value> + <div className="flex flex-col gap-0.5"> + <TextLink to={queuePath}>{task.queue.name}</TextLink> + <Paragraph variant="extra-small" className="text-text-dimmed"> + Concurrency: {task.queue.concurrencyLimit ?? "Unlimited"} + {task.queue.paused ? " · Paused" : ""} + </Paragraph> + <QueueSidebarStats + live={queueMetrics.live} + ids={queueMetrics.ids} + queueName={task.queue.name} + defaultPeriod="7d" + /> + </div> + </Property.Value> + </Property.Item> + ) : null} </Property.Table> {scheduleList && sortedSchedules.length === 0 ? ( <div className="mt-4"> diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx index ac765406ce1..12227dd66fd 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx @@ -11,6 +11,7 @@ import { PageBody, PageContainer } from "~/components/layout/AppLayout"; import { DirectionSchema, ListPagination } from "~/components/ListPagination"; import { LinkButton } from "~/components/primitives/Buttons"; import { ChartCard } from "~/components/primitives/charts/ChartCard"; +import { TabButton, TabContainer } from "~/components/primitives/Tabs"; import { ChartSyncProvider } from "~/components/primitives/charts/ChartSyncContext"; import { useZoomToTimeFilter } from "~/hooks/useZoomToTimeFilter"; import { Chart, type ChartConfig } from "~/components/primitives/charts/ChartCompound"; @@ -30,10 +31,18 @@ import { import { Spinner } from "~/components/primitives/Spinner"; import { TextLink } from "~/components/primitives/TextLink"; import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; +import { + QUEUE_METRIC_COLORS, + QueueMetricChart, + QueueSidebarStats, + type QueueLiveCounts, + type QueueMetricIds, +} from "~/components/queues/QueueMetricCards"; import { $replica } from "~/db.server"; import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; +import { useSearchParams } from "~/hooks/useSearchParam"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { NextRunListPresenter } from "~/presenters/v3/NextRunListPresenter.server"; @@ -47,11 +56,14 @@ import { requireUser } from "~/services/session.server"; import { EnvironmentParamSchema, v3EnvironmentPath, + v3QueuePath, v3QueuesPath, v3TestTaskPath, } from "~/utils/pathBuilder"; import { parseFiniteInt } from "~/utils/searchParams"; import { NewRunsButton, TaskRunsList } from "~/components/runs/v3/TaskRunsList"; +import { canAccessQueueMetricsUi } from "~/v3/canAccessQueueMetricsUi.server"; +import { engine } from "~/v3/runEngine.server"; export const meta: MetaFunction<typeof loader> = ({ data }) => { const slug = (data as { task?: TaskDetail | null } | undefined)?.task?.slug; @@ -97,6 +109,28 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { if (!task) throw new Response("Task not found", { status: 404 }); + // Live queue counts (two O(1) Redis reads) shown in the sidebar; history charts fetch + // client-side through the metric resource. Flag off = no extra reads at all. + let queueMetrics: { live: QueueLiveCounts; ids: QueueMetricIds } | null = null; + if (task.queue && (await canAccessQueueMetricsUi({ userId, organizationSlug }))) { + const queueName = task.queue.name; + const [lengths, concurrency] = await Promise.all([ + engine.lengthOfQueues(environment, [queueName]), + engine.currentConcurrencyOfQueues(environment, [queueName]), + ]); + queueMetrics = { + live: { + queued: lengths?.[queueName] ?? 0, + running: concurrency?.[queueName] ?? 0, + }, + ids: { + organizationId: project.organizationId, + projectId: project.id, + environmentId: environment.id, + }, + }; + } + const time = timeFilterFromTo({ period, from, to, defaultPeriod: "7d" }); const activity = presenter @@ -129,11 +163,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { task, activity, runList, + queueMetrics, }); }; export default function Page() { - const { task, activity, runList } = useTypedLoaderData<typeof loader>(); + const { task, activity, runList, queueMetrics } = useTypedLoaderData<typeof loader>(); const zoomToTimeFilter = useZoomToTimeFilter(); const organization = useOrganization(); const project = useProject(); @@ -144,6 +179,36 @@ export default function Page() { taskIdentifier: task.slug, }); const queuesPath = v3QueuesPath(organization, project, environment); + const queuePath = task.queue + ? v3QueuePath(organization, project, environment, { friendlyId: task.queue.friendlyId }) + : undefined; + + const { value } = useSearchParams(); + const timeRange = { + period: value("period") ?? null, + from: value("from") ?? null, + to: value("to") ?? null, + }; + + const runsByStatusChart = ( + <Suspense fallback={<ActivityChartSkeleton />}> + <TypedAwait resolve={activity} errorElement={<ActivityChartSkeleton />}> + {(result) => <ActivityChart activity={result} />} + </TypedAwait> + </Suspense> + ); + + const activityPanel = + queueMetrics && task.queue ? ( + <TaskActivityCard + runsByStatusChart={runsByStatusChart} + queueName={task.queue.name} + ids={queueMetrics.ids} + timeRange={timeRange} + /> + ) : ( + <ChartCard title="Runs by status">{runsByStatusChart}</ChartCard> + ); // New-runs banner state is lifted here so the button can live in the top bar, // while the count/action originate from the live-reload hook inside the @@ -190,15 +255,7 @@ export default function Page() { {/* Activity chart */} <ResizablePanel id="task-activity" min="220px" default="320px"> <div className="flex h-full min-h-0 flex-col overflow-hidden bg-background p-2"> - <ChartSyncProvider onZoom={zoomToTimeFilter}> - <ChartCard title="Runs by status"> - <Suspense fallback={<ActivityChartSkeleton />}> - <TypedAwait resolve={activity} errorElement={<ActivityChartSkeleton />}> - {(result) => <ActivityChart activity={result} />} - </TypedAwait> - </Suspense> - </ChartCard> - </ChartSyncProvider> + <ChartSyncProvider onZoom={zoomToTimeFilter}>{activityPanel}</ChartSyncProvider> </div> </ResizablePanel> @@ -231,7 +288,13 @@ export default function Page() { <ResizableHandle id="task-detail-handle" /> <ResizablePanel id="task-detail" min="280px" default="380px" max="500px" isStaticAtRest> - <TaskDetailSidebar task={task} testPath={testPath} queuesPath={queuesPath} /> + <TaskDetailSidebar + task={task} + testPath={testPath} + queuesPath={queuesPath} + queuePath={queuePath} + queueMetrics={queueMetrics} + /> </ResizablePanel> </ResizablePanelGroup> </PageBody> @@ -243,10 +306,14 @@ function TaskDetailSidebar({ task, testPath, queuesPath, + queuePath, + queueMetrics, }: { task: TaskDetail; testPath: string; queuesPath: string; + queuePath: string | undefined; + queueMetrics: { live: QueueLiveCounts; ids: QueueMetricIds } | null; }) { const showExportName = task.exportName && task.exportName !== task.slug; const retrySummary = formatRetrySummary(task.retry); @@ -320,11 +387,21 @@ function TaskDetailSidebar({ <Property.Label>Queue</Property.Label> <Property.Value> <div className="flex flex-col gap-0.5"> - <TextLink to={queuesPath}>{task.queue.name}</TextLink> + <TextLink to={queueMetrics && queuePath ? queuePath : queuesPath}> + {task.queue.name} + </TextLink> <Paragraph variant="extra-small" className="text-text-dimmed"> Concurrency: {task.queue.concurrencyLimit ?? "Unlimited"} {task.queue.paused ? " · Paused" : ""} </Paragraph> + {queueMetrics ? ( + <QueueSidebarStats + live={queueMetrics.live} + ids={queueMetrics.ids} + queueName={task.queue.name} + defaultPeriod="7d" + /> + ) : null} </div> </Property.Value> </Property.Item> @@ -384,6 +461,58 @@ function formatRetrySummary(retry: TaskDetail["retry"]): string { return `${retry.maxAttempts} attempts`; } +// Activity panel for tasks with a queue: tabs in the card header switch between the +// runs-by-status chart and the queue backlog, so we never add a second chart alongside. +function TaskActivityCard({ + runsByStatusChart, + queueName, + ids, + timeRange, +}: { + runsByStatusChart: React.ReactNode; + queueName: string; + ids: QueueMetricIds; + timeRange: { period: string | null; from: string | null; to: string | null }; +}) { + const [view, setView] = useState<"runs" | "queue">("runs"); + return ( + <ChartCard + title={ + <TabContainer> + <TabButton + isActive={view === "runs"} + layoutId="task-activity-view" + onClick={() => setView("runs")} + > + Runs by status + </TabButton> + <TabButton + isActive={view === "queue"} + layoutId="task-activity-view" + onClick={() => setView("queue")} + > + Queue backlog + </TabButton> + </TabContainer> + } + > + {view === "queue" ? ( + <QueueMetricChart + query={`SELECT timeBucket() AS t, max(max_queued) AS queued\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + fillGaps + ids={ids} + timeRange={timeRange} + queueName={queueName} + defaultPeriod="7d" + series={[{ key: "queued", label: "Queued", color: QUEUE_METRIC_COLORS.queued }]} + /> + ) : ( + runsByStatusChart + )} + </ChartCard> + ); +} + function ActivityChart({ activity }: { activity: TaskActivity }) { const chartConfig: ChartConfig = useMemo(() => { const cfg: ChartConfig = {}; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.usage/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.usage/route.tsx index 48d719932cf..da2359a3057 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.usage/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.usage/route.tsx @@ -7,7 +7,8 @@ import { redirect, typeddefer, useTypedLoaderData } from "remix-typedjson"; import { URL } from "url"; import { UsageBar } from "~/components/billing/UsageBar"; import { getUsageBarBillingLimitDollars } from "~/components/billing/billingAlertsFormat"; -import { PageBody, PageContainer } from "~/components/layout/AppLayout"; +import { PageContainer } from "~/components/layout/AppLayout"; +import { MetricsLayout } from "~/components/layout/MetricsLayout"; import { Card } from "~/components/primitives/charts/Card"; import type { ChartConfig } from "~/components/primitives/charts/Chart"; import { Chart } from "~/components/primitives/charts/ChartCompound"; @@ -129,13 +130,12 @@ export default function Page() { <NavBar> <PageTitle title="Usage" /> </NavBar> - <PageBody scrollable={true} className="p-0"> - <div className="flex flex-col gap-6"> - <div> + <MetricsLayout.Root> + <MetricsLayout.Filters> + <div className="flex items-center gap-2"> <Select name="month" placeholder="Select a month" - className="m-3" defaultValue={month} items={months.map((date) => ({ label: monthDateFormatter.format(date), @@ -156,47 +156,93 @@ export default function Page() { )) } </Select> - {promoCredits && ( - <div className="flex flex-col gap-1 border-t border-grid-dimmed p-3"> - <div className="flex items-end gap-8"> - <div className="flex flex-col gap-1"> - <Header2 className="whitespace-nowrap">Credits</Header2> - <p className="whitespace-nowrap text-3xl font-medium text-text-bright"> - {formatCurrency(promoCredits.remainingCents / 100, false)} - </p> - </div> - <div className="flex w-full flex-1 flex-col gap-1 pb-1"> - <div className="h-2 w-full overflow-hidden rounded-full bg-charcoal-700"> - <div - className="h-full rounded-full bg-blue-500" - style={{ - width: `${ - promoCredits.grantedCents > 0 - ? Math.min( - 100, - Math.max( - 0, - (promoCredits.remainingCents / promoCredits.grantedCents) * 100 - ) + </div> + </MetricsLayout.Filters> + + <MetricsLayout.Grid columns={{ base: 1 }}> + {promoCredits && ( + <div className="flex flex-col gap-1"> + <div className="flex items-end gap-8"> + <div className="flex flex-col gap-1"> + <Header2 className="whitespace-nowrap">Credits</Header2> + <p className="whitespace-nowrap text-3xl font-medium text-text-bright"> + {formatCurrency(promoCredits.remainingCents / 100, false)} + </p> + </div> + <div className="flex w-full flex-1 flex-col gap-1 pb-1"> + <div className="h-2 w-full overflow-hidden rounded-full bg-charcoal-700"> + <div + className="h-full rounded-full bg-blue-500" + style={{ + width: `${ + promoCredits.grantedCents > 0 + ? Math.min( + 100, + Math.max( + 0, + (promoCredits.remainingCents / promoCredits.grantedCents) * 100 ) - : 0 - }%`, - }} - /> - </div> - <Paragraph variant="extra-small" className="text-text-dimmed"> - {formatCurrency(promoCredits.remainingCents / 100, false)} of{" "} - {formatCurrency(promoCredits.grantedCents / 100, false)} remaining - {promoCredits.expiresAt - ? ` · expires ${creditExpiryFormatter.format(new Date(promoCredits.expiresAt))}` - : ""} - </Paragraph> + ) + : 0 + }%`, + }} + /> </div> + <Paragraph variant="extra-small" className="text-text-dimmed"> + {formatCurrency(promoCredits.remainingCents / 100, false)} of{" "} + {formatCurrency(promoCredits.grantedCents / 100, false)} remaining + {promoCredits.expiresAt + ? ` · expires ${creditExpiryFormatter.format(new Date(promoCredits.expiresAt))}` + : ""} + </Paragraph> </div> </div> - )} - <div className="flex w-full flex-col gap-2 border-t border-grid-dimmed p-3"> - <Suspense fallback={<Spinner />}> + </div> + )} + <div className="flex w-full flex-col gap-2"> + <Suspense fallback={<Spinner />}> + <Await + resolve={usage} + errorElement={ + <div className="flex min-h-40 items-center justify-center"> + <Paragraph variant="small">Failed to load graph.</Paragraph> + </div> + } + > + {(usage) => ( + <div className="flex items-end gap-8"> + <div className="flex flex-col gap-1"> + <Header2 className="whitespace-nowrap"> + {isCurrentMonth ? "Month-to-date" : "Usage"} + </Header2> + <p className="whitespace-nowrap text-3xl font-medium text-text-bright"> + {formatCurrency(usage.overall.current, false)} + </p> + </div> + <UsageBar + current={usage.overall.current} + isPaying={currentPlan?.v3Subscription?.isPaying ?? false} + tierLimit={isCurrentMonth && !isEnterprise ? planLimitCents / 100 : undefined} + billingLimit={billingLimitDollars} + /> + </div> + )} + </Await> + </Suspense> + </div> + </MetricsLayout.Grid> + + <MetricsLayout.Grid> + <Card> + <Card.Header>Usage by day</Card.Header> + <Card.Content> + <Suspense + fallback={ + <div className="flex min-h-40 items-center justify-center"> + <Spinner /> + </div> + } + > <Await resolve={usage} errorElement={ @@ -205,138 +251,89 @@ export default function Page() { </div> } > - {(usage) => ( - <div className="flex items-end gap-8"> - <div className="flex flex-col gap-1"> - <Header2 className="whitespace-nowrap"> - {isCurrentMonth ? "Month-to-date" : "Usage"} - </Header2> - <p className="whitespace-nowrap text-3xl font-medium text-text-bright"> - {formatCurrency(usage.overall.current, false)} - </p> - </div> - <UsageBar - current={usage.overall.current} - isPaying={currentPlan?.v3Subscription?.isPaying ?? false} - tierLimit={ - isCurrentMonth && !isEnterprise ? planLimitCents / 100 : undefined - } - billingLimit={billingLimitDollars} - /> - </div> - )} + {(u) => <UsageChart data={u.timeSeries} />} </Await> </Suspense> - </div> - </div> - - <div className="px-3"> - <Card> - <Card.Header>Usage by day</Card.Header> - <Card.Content> - <Suspense - fallback={ - <div className="flex min-h-40 items-center justify-center"> - <Spinner /> - </div> - } - > - <Await - resolve={usage} - errorElement={ - <div className="flex min-h-40 items-center justify-center"> - <Paragraph variant="small">Failed to load graph.</Paragraph> - </div> - } - > - {(u) => <UsageChart data={u.timeSeries} />} - </Await> - </Suspense> - </Card.Content> - </Card> - </div> - <div> - <Header2 spacing className="pl-3"> - Tasks - </Header2> - <Suspense fallback={<Spinner />}> - <Await - resolve={tasks} - errorElement={ - <div className="flex min-h-40 items-center justify-center"> - <Paragraph variant="small">Failed to load.</Paragraph> - </div> - } - > - {(tasks) => { - return ( - <> - <Table> - <TableHeader> + </Card.Content> + </Card> + </MetricsLayout.Grid> + <MetricsLayout.Content> + <Header2 className="pl-3">Tasks</Header2> + <Suspense fallback={<Spinner />}> + <Await + resolve={tasks} + errorElement={ + <div className="flex min-h-40 items-center justify-center"> + <Paragraph variant="small">Failed to load.</Paragraph> + </div> + } + > + {(tasks) => { + return ( + <> + <Table> + <TableHeader> + <TableRow> + <TableHeaderCell>Task</TableHeaderCell> + <TableHeaderCell alignment="right">Runs</TableHeaderCell> + <TableHeaderCell alignment="right">Average duration</TableHeaderCell> + <TableHeaderCell alignment="right">Average cost</TableHeaderCell> + <TableHeaderCell alignment="right">Total duration</TableHeaderCell> + <TableHeaderCell alignment="right">Total cost</TableHeaderCell> + </TableRow> + </TableHeader> + <TableBody> + {tasks.length === 0 ? ( <TableRow> - <TableHeaderCell>Task</TableHeaderCell> - <TableHeaderCell alignment="right">Runs</TableHeaderCell> - <TableHeaderCell alignment="right">Average duration</TableHeaderCell> - <TableHeaderCell alignment="right">Average cost</TableHeaderCell> - <TableHeaderCell alignment="right">Total duration</TableHeaderCell> - <TableHeaderCell alignment="right">Total cost</TableHeaderCell> + <TableCell colSpan={6}> + <div className="flex items-center justify-center py-8"> + <Paragraph variant="base/bright">No runs for this period</Paragraph> + </div> + </TableCell> </TableRow> - </TableHeader> - <TableBody> - {tasks.length === 0 ? ( - <TableRow> - <TableCell colSpan={6}> - <div className="flex items-center justify-center py-8"> - <Paragraph variant="base/bright"> - No runs for this period - </Paragraph> - </div> + ) : ( + tasks.map((task) => ( + <TableRow key={task.taskIdentifier}> + <TableCell>{task.taskIdentifier}</TableCell> + <TableCell alignment="right" className="tabular-nums"> + {formatNumber(task.runCount)} + </TableCell> + <TableCell alignment="right"> + {formatDurationMilliseconds(task.averageDuration, { + style: "short", + })} + </TableCell> + <TableCell alignment="right" className="tabular-nums"> + {formatCurrencyAccurate(task.averageCost)} + </TableCell> + <TableCell alignment="right" className="tabular-nums"> + {formatDurationMilliseconds(task.totalDuration, { + style: "short", + })} + </TableCell> + <TableCell alignment="right" className="tabular-nums"> + {formatCurrencyAccurate(task.totalCost)} </TableCell> </TableRow> - ) : ( - tasks.map((task) => ( - <TableRow key={task.taskIdentifier}> - <TableCell>{task.taskIdentifier}</TableCell> - <TableCell alignment="right" className="tabular-nums"> - {formatNumber(task.runCount)} - </TableCell> - <TableCell alignment="right"> - {formatDurationMilliseconds(task.averageDuration, { - style: "short", - })} - </TableCell> - <TableCell alignment="right" className="tabular-nums"> - {formatCurrencyAccurate(task.averageCost)} - </TableCell> - <TableCell alignment="right" className="tabular-nums"> - {formatDurationMilliseconds(task.totalDuration, { - style: "short", - })} - </TableCell> - <TableCell alignment="right" className="tabular-nums"> - {formatCurrencyAccurate(task.totalCost)} - </TableCell> - </TableRow> - )) - )} - </TableBody> - </Table> - <InfoPanel - icon={InformationCircleIcon} - variant="minimal" - panelClassName="max-w-full" - > - Dev environment runs are excluded from the usage data above, since they do - not have an associated compute cost. - </InfoPanel> - </> - ); - }} - </Await> - </Suspense> - </div> - </div> - </PageBody> + )) + )} + </TableBody> + </Table> + <InfoPanel + icon={InformationCircleIcon} + variant="minimal" + panelClassName="max-w-full" + > + Dev environment runs are excluded from the usage data above, since they do not + have an associated compute cost. + </InfoPanel> + </> + ); + }} + </Await> + </Suspense> + </MetricsLayout.Content> + </MetricsLayout.Root> </PageContainer> ); } diff --git a/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.ts b/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.ts index 908e8f449a0..9717aa2b942 100644 --- a/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.ts +++ b/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.ts @@ -6,6 +6,7 @@ import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; import { engine } from "~/v3/runEngine.server"; import { updateEnvConcurrencyLimits } from "~/v3/runQueue.server"; +import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server"; const ParamsSchema = z.object({ environmentId: z.string(), @@ -46,6 +47,9 @@ export async function action({ request, params }: ActionFunctionArgs) { await updateEnvConcurrencyLimits(environment); + // Percent-based queue overrides follow the environment limit automatically. + await concurrencySystem.queues.recalculatePercentLimits(environment); + // Org max-concurrency changed too, which is embedded in every env of the org; invalidating // the org drops the env/authEnv rows for all of them (including this env). controlPlaneResolver.invalidateOrganization(environment.organizationId); diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts index c99637a0d10..403df19fe37 100644 --- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts @@ -5,6 +5,7 @@ import { prisma } from "~/db.server"; import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; import { updateEnvConcurrencyLimits } from "~/v3/runQueue.server"; +import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server"; const ParamsSchema = z.object({ organizationId: z.string(), @@ -82,6 +83,12 @@ export async function action({ request, params }: ActionFunctionArgs) { }); await updateEnvConcurrencyLimits({ ...modifiedEnvironment, organization }); + + // Percent-based queue overrides follow the environment limit automatically. + await concurrencySystem.queues.recalculatePercentLimits({ + ...modifiedEnvironment, + organization, + }); } // Org + every affected env's concurrency changed; one org invalidation covers them all. diff --git a/apps/webapp/app/routes/admin.api.v1.queue-metrics.ts b/apps/webapp/app/routes/admin.api.v1.queue-metrics.ts new file mode 100644 index 00000000000..69e4e8c1fac --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.queue-metrics.ts @@ -0,0 +1,45 @@ +import { type ActionFunctionArgs, type LoaderFunctionArgs, json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; +import { + probeQueueMetricsStreams, + readQueueMetricsControls, + writeQueueMetricsControls, +} from "~/v3/queueMetrics.server"; + +export async function loader({ request }: LoaderFunctionArgs) { + await requireAdminApiRequest(request); + const [controls, streams] = await Promise.all([ + readQueueMetricsControls(), + probeQueueMetricsStreams(), + ]); + return json({ controls, streams }); +} + +const BodySchema = z.object({ + enabled: z.boolean().optional(), + sampleRate: z.number().min(0).max(1).optional(), +}); + +export async function action({ request }: ActionFunctionArgs) { + await requireAdminApiRequest(request); + + if (request.method !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + let body: unknown; + try { + body = await request.json(); + } catch { + return json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const parsed = BodySchema.safeParse(body); + if (!parsed.success) { + return json({ error: "Invalid payload", details: parsed.error.issues }, { status: 400 }); + } + + await writeQueueMetricsControls(parsed.data); + return json({ ok: true, controls: await readQueueMetricsControls() }); +} diff --git a/apps/webapp/app/routes/admin.queue-metrics.tsx b/apps/webapp/app/routes/admin.queue-metrics.tsx new file mode 100644 index 00000000000..6deaedce66e --- /dev/null +++ b/apps/webapp/app/routes/admin.queue-metrics.tsx @@ -0,0 +1,190 @@ +import { useFetcher, useRevalidator } from "@remix-run/react"; +import { json } from "@remix-run/server-runtime"; +import { useEffect, useState } from "react"; +import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { z } from "zod"; +import { Button } from "~/components/primitives/Buttons"; +import { Callout } from "~/components/primitives/Callout"; +import { Header1, Header2 } from "~/components/primitives/Headers"; +import { Input } from "~/components/primitives/Input"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { + Table, + TableBody, + TableCell, + TableHeader, + TableHeaderCell, + TableRow, +} from "~/components/primitives/Table"; +import { dashboardAction, dashboardLoader } from "~/services/routeBuilders/dashboardBuilder"; +import { + probeQueueMetricsStreams, + readQueueMetricsControls, + writeQueueMetricsControls, +} from "~/v3/queueMetrics.server"; + +export const loader = dashboardLoader({ authorization: { requireSuper: true } }, async () => { + const [controls, streams] = await Promise.all([ + readQueueMetricsControls(), + probeQueueMetricsStreams(), + ]); + return typedjson({ controls, streams }); +}); + +const BodySchema = z.object({ + enabled: z.boolean().optional(), + sampleRate: z.number().min(0).max(1).optional(), +}); + +export const action = dashboardAction( + { authorization: { requireSuper: true } }, + async ({ request }) => { + let body: unknown; + try { + body = await request.json(); + } catch { + return json({ error: "Invalid JSON body" }, { status: 400 }); + } + const parsed = BodySchema.safeParse(body); + if (!parsed.success) { + return json({ error: "Invalid payload" }, { status: 400 }); + } + await writeQueueMetricsControls(parsed.data); + return json({ success: true }); + } +); + +export default function AdminQueueMetricsRoute() { + const { controls, streams } = useTypedLoaderData<typeof loader>(); + const saveFetcher = useFetcher<{ success?: boolean; error?: string }>(); + const revalidator = useRevalidator(); + + const [enabled, setEnabled] = useState(controls.enabled); + const [sampleRate, setSampleRate] = useState(String(controls.sampleRate)); + const [error, setError] = useState<string | null>(null); + + useEffect(() => { + setEnabled(controls.enabled); + setSampleRate(String(controls.sampleRate)); + }, [controls.enabled, controls.sampleRate]); + + useEffect(() => { + if (saveFetcher.data?.success) { + setError(null); + revalidator.revalidate(); + } else if (saveFetcher.data?.error) { + setError(saveFetcher.data.error); + } + }, [saveFetcher.data]); + + const isSaving = saveFetcher.state === "submitting"; + + const handleSave = () => { + const rate = Number(sampleRate); + if (!Number.isFinite(rate) || rate < 0 || rate > 1) { + setError("Sample rate must be a number between 0 and 1"); + return; + } + saveFetcher.submit(JSON.stringify({ enabled, sampleRate: rate }), { + method: "POST", + encType: "application/json", + }); + }; + + const totalLag = streams.reduce((sum, s) => sum + (s.lag ?? 0), 0); + const lagUnknownCount = streams.filter((s) => s.lag === null).length; + + return ( + <main className="flex h-full min-w-0 flex-1 flex-col overflow-y-auto px-4 pb-4 lg:order-last"> + <div className="max-w-2xl space-y-4 py-4"> + <Header1>Queue metrics ingest</Header1> + <Callout variant="warning"> + Live controls for the queue-metrics ingest pipeline on the run-queue Redis. Changes take + effect within ~10s across all instances (no redeploy). Watch EngineCPU on the run-queue + Redis when enabling or raising the sample rate. + </Callout> + + <div className="space-y-3 rounded-md border border-grid-bright p-4"> + <Header2>Controls</Header2> + <label className="flex items-center gap-2 text-sm text-text-bright"> + <input + type="checkbox" + checked={enabled} + onChange={(e) => setEnabled(e.target.checked)} + /> + Emission enabled <span className="text-text-dimmed">(queue_metrics:enabled)</span> + </label> + <div className="flex flex-col gap-1"> + <label className="text-sm text-text-dimmed"> + Gauge sample rate 0–1 (queue_metrics:gauge_sample_rate); default{" "} + {controls.sampleRateDefault} + </label> + <Input + type="number" + min={0} + max={1} + step={0.05} + value={sampleRate} + onChange={(e) => setSampleRate(e.target.value)} + className="w-32" + /> + </div> + {error && <Callout variant="error">{error}</Callout>} + <div className="flex justify-end"> + <Button variant="primary/small" onClick={handleSave} disabled={isSaving}> + {isSaving ? "Saving..." : "Save controls"} + </Button> + </div> + </div> + + <div className="space-y-3 rounded-md border border-grid-bright p-4"> + <div className="flex items-center justify-between"> + <Header2>Stream health{totalLag > 0 ? ` (lag ${totalLag})` : ""}</Header2> + <Button + variant="tertiary/small" + onClick={() => revalidator.revalidate()} + disabled={revalidator.state === "loading"} + > + Refresh + </Button> + </div> + <Paragraph variant="extra-small"> + Depth = entries buffered in the shard stream; Lag = entries not yet delivered to the + consumer group (rising = consumer falling behind; "unknown" = entries were trimmed past + the group, i.e. data was lost); Pending = unacked entries. Gauges and counters share one + stream family on the metrics Redis. + </Paragraph> + {lagUnknownCount > 0 && ( + <Callout variant="error"> + Lag is unknown on {lagUnknownCount} shard{lagUnknownCount === 1 ? "" : "s"}: entries + were trimmed past the consumer group's read position, so stream data was lost. Check + consumer health. + </Callout> + )} + <Table> + <TableHeader> + <TableRow> + <TableHeaderCell>Stream</TableHeaderCell> + <TableHeaderCell>Shard</TableHeaderCell> + <TableHeaderCell alignment="right">Depth</TableHeaderCell> + <TableHeaderCell alignment="right">Lag</TableHeaderCell> + <TableHeaderCell alignment="right">Pending</TableHeaderCell> + </TableRow> + </TableHeader> + <TableBody> + {streams.map((s) => ( + <TableRow key={`${s.stream}-${s.shard}`}> + <TableCell>{s.stream}</TableCell> + <TableCell>{s.shard}</TableCell> + <TableCell alignment="right">{s.depth}</TableCell> + <TableCell alignment="right">{s.lag ?? "unknown"}</TableCell> + <TableCell alignment="right">{s.pending}</TableCell> + </TableRow> + ))} + </TableBody> + </Table> + </div> + </div> + </main> + ); +} diff --git a/apps/webapp/app/routes/admin.tsx b/apps/webapp/app/routes/admin.tsx index 34ba6c62ca1..2cd7cbfd1ec 100644 --- a/apps/webapp/app/routes/admin.tsx +++ b/apps/webapp/app/routes/admin.tsx @@ -34,6 +34,10 @@ export default function Page() { label: "Global Feature Flags", to: "/admin/feature-flags", }, + { + label: "Queue Metrics", + to: "/admin/queue-metrics", + }, { label: "Notifications", to: "/admin/notifications", diff --git a/apps/webapp/app/routes/api.v1.query.schema.ts b/apps/webapp/app/routes/api.v1.query.schema.ts index 3e95d16818d..e28f9306278 100644 --- a/apps/webapp/app/routes/api.v1.query.schema.ts +++ b/apps/webapp/app/routes/api.v1.query.schema.ts @@ -1,7 +1,8 @@ import { json } from "@remix-run/server-runtime"; import type { ColumnSchema, TableSchema } from "@internal/tsql"; import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; -import { querySchemas } from "~/v3/querySchemas"; +import { listableQuerySchemas } from "~/v3/querySchemas"; +import { env } from "~/env.server"; function serializeColumn(col: ColumnSchema) { const result: Record<string, unknown> = { @@ -51,7 +52,9 @@ export const loader = createLoaderApiRoute( }, }, async () => { - const tables = querySchemas.map(serializeTable); + const tables = listableQuerySchemas({ + includeQueueMetrics: env.QUEUE_METRICS_QUERY_TABLES_VISIBLE === "1", + }).map(serializeTable); return json({ tables }); } ); diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts index 3296e7fa782..8980aaeeb47 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts @@ -4,11 +4,23 @@ import { z } from "zod"; import { toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server"; import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server"; +import { + MAX_QUEUE_OVERRIDE_PERCENT, + MIN_QUEUE_OVERRIDE_PERCENT, +} from "~/v3/services/concurrencySystem.server"; -const BodySchema = z.object({ - type: RetrieveQueueType.default("id"), - concurrencyLimit: z.number().int().min(0).max(100000), -}); +const BodySchema = z + .object({ + type: RetrieveQueueType.default("id"), + // Absolute concurrency limit. Backwards compatible with existing callers. + concurrencyLimit: z.number().int().min(0).max(100000).optional(), + // Percentage of the environment's maximum concurrency limit (0 < percent <= 100). + // Stored as the source of truth; the absolute limit is materialized from it. + percent: z.number().gt(MIN_QUEUE_OVERRIDE_PERCENT).max(MAX_QUEUE_OVERRIDE_PERCENT).optional(), + }) + .refine((body) => (body.concurrencyLimit === undefined) !== (body.percent === undefined), { + message: "Provide exactly one of `concurrencyLimit` or `percent`", + }); const route = createActionApiRoute( { @@ -26,8 +38,11 @@ const route = createActionApiRoute( name: decodeURIComponent(params.queueParam).replace(/%2F/g, "/"), }; + const override = + body.percent !== undefined ? { percent: body.percent } : { limit: body.concurrencyLimit! }; + return concurrencySystem.queues - .overrideQueueConcurrencyLimit(authentication.environment, input, body.concurrencyLimit) + .overrideQueueConcurrencyLimit(authentication.environment, input, override) .match( (queue) => { return json( @@ -51,6 +66,10 @@ const route = createActionApiRoute( case "queue_not_found": { return json({ error: "Queue not found" }, { status: 404 }); } + case "invalid_override": + case "concurrency_limit_exceeds_maximum": { + return json({ error: error.message }, { status: 400 }); + } case "queue_update_failed": { return json({ error: "Failed to update queue concurrency limit" }, { status: 500 }); } diff --git a/apps/webapp/app/routes/api.v1.reports.$key.ts b/apps/webapp/app/routes/api.v1.reports.$key.ts new file mode 100644 index 00000000000..bcb504458cf --- /dev/null +++ b/apps/webapp/app/routes/api.v1.reports.$key.ts @@ -0,0 +1,93 @@ +import { json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { ReportPresenter } from "~/presenters/v3/reports/ReportPresenter.server"; +import { isReportKey, REPORT_KEYS } from "~/presenters/v3/reports/report-registry"; +import { renderReportAnsi, renderReportMarkdown } from "~/presenters/v3/reports/renderMarkdown"; +import { logger } from "~/services/logger.server"; +import { createLoaderApiRoute, everyResource } from "~/services/routeBuilders/apiBuilder.server"; + +const ParamsSchema = z.object({ + key: z.string(), +}); + +/** + * The query tables the reports read. Authorize per-table (like api.v1.query.ts) rather than + * the permissive `{ type: "query", id: "all" }`: a JWT must be scoped to every table a report + * touches, so a token scoped to only some tables can't fetch a report that reads others. This + * is the union across reports; `health` reads all three. + */ +const REPORT_QUERY_TABLES = ["runs", "env_metrics", "queue_metrics"] as const; + +/** Canonical shorthand ("1h" / "30m" / "7d") with an upper bound, so the public API rejects + * garbage and absurd ranges (e.g. "999999999d") itself rather than relying on downstream clip. */ +const UNIT_MS: Record<string, number> = { s: 1e3, m: 6e4, h: 36e5, d: 864e5, w: 6048e5 }; +const MAX_PERIOD_MS = 90 * UNIT_MS.d; +const PeriodSchema = z + .string() + .regex(/^[1-9]\d*[smhdw]$/, "period must be a shorthand like '1h', '30m', or '7d'") + .refine( + (p) => Number(p.slice(0, -1)) * UNIT_MS[p.slice(-1)] <= MAX_PERIOD_MS, + "period is too large (max 90d)" + ); + +const SearchParamsSchema = z.object({ + period: PeriodSchema.optional(), + // markdown (default) for CLI/MCP · json (the raw VM) for web · ansi for a colour terminal. + format: z.enum(["markdown", "json", "ansi"]).default("markdown"), +}); + +export const loader = createLoaderApiRoute( + { + params: ParamsSchema, + searchParams: SearchParamsSchema, + // The MCP `get_report` tool calls this with a scoped JWT (read:query), so JWT auth + // must be allowed — same as api.v1.query.ts. + allowJWT: true, + findResource: async () => 1, // dummy — report key validated in the handler + authorization: { + action: "read", + // Per-table, not `id: "all"`: a JWT must be scoped to every query table the report reads. + resource: () => everyResource(REPORT_QUERY_TABLES.map((id) => ({ type: "query", id }))), + }, + }, + async ({ params, searchParams, authentication }) => { + if (!isReportKey(params.key)) { + return json( + { error: `Unknown report "${params.key}". Available: ${REPORT_KEYS.join(", ")}.` }, + { status: 404 } + ); + } + + try { + const presenter = new ReportPresenter(); + const vm = await presenter.call({ + environment: authentication.environment, + key: params.key, + period: searchParams.period, + }); + + if (!vm) { + return json({ error: `Unknown report "${params.key}".` }, { status: 404 }); + } + + if (searchParams.format === "json") { + return json(vm, { status: 200 }); + } + + if (searchParams.format === "ansi") { + return new Response(renderReportAnsi(vm), { + status: 200, + headers: { "Content-Type": "text/plain; charset=utf-8" }, + }); + } + + return new Response(renderReportMarkdown(vm), { + status: 200, + headers: { "Content-Type": "text/markdown; charset=utf-8" }, + }); + } catch (error) { + logger.error("Failed to render report", { error, key: params.key }); + return json({ error: "Something went wrong, please try again." }, { status: 500 }); + } + } +); diff --git a/apps/webapp/app/routes/resources.metric.tsx b/apps/webapp/app/routes/resources.metric.tsx index d456ba1ce1b..5bf0ed693ad 100644 --- a/apps/webapp/app/routes/resources.metric.tsx +++ b/apps/webapp/app/routes/resources.metric.tsx @@ -50,6 +50,8 @@ const MetricWidgetQuery = z.object({ operations: z.array(z.string()).optional(), providers: z.array(z.string()).optional(), tags: z.array(z.string()).optional(), + // Opt into server-side gap fill (carry-forward for gauges, zero-fill for counters). + fillGaps: z.boolean().optional(), }); export const action = async ({ request }: ActionFunctionArgs) => { @@ -85,6 +87,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { operations, providers, tags: _tags, + fillGaps, } = submission.data; // Check they should be able to access it @@ -122,6 +125,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { promptVersions, operations, providers, + fillGaps, // Set higher concurrency if many widgets are on screen at once customOrgConcurrencyLimit: env.METRIC_WIDGET_DEFAULT_ORG_CONCURRENCY_LIMIT, }); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-generate.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-generate.tsx index c1626b966d2..36ead3cc856 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-generate.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-generate.tsx @@ -8,7 +8,7 @@ import type { AITimeFilter } from "~/routes/_app.orgs.$organizationSlug.projects import { requireUserId } from "~/services/session.server"; import { EnvironmentParamSchema } from "~/utils/pathBuilder"; import { AIQueryService } from "~/v3/services/aiQueryService.server"; -import { querySchemas } from "~/v3/querySchemas"; +import { listableQuerySchemas } from "~/v3/querySchemas"; const RequestSchema = z.object({ prompt: z.string().min(1, "Prompt is required"), @@ -85,7 +85,9 @@ export async function action({ request, params }: ActionFunctionArgs) { const { prompt, mode, currentQuery } = submission.data; const service = new AIQueryService( - querySchemas, + listableQuerySchemas({ + includeQueueMetrics: env.QUEUE_METRICS_QUERY_TABLES_VISIBLE === "1", + }), openai(env.AI_RUN_FILTER_MODEL ?? "gpt-4o-mini") ); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx index 4c8f77a582a..4e37354c831 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx @@ -7,6 +7,7 @@ import { ClockIcon, CloudArrowDownIcon, EnvelopeIcon, + ExclamationTriangleIcon, GlobeAltIcon, KeyIcon, QueueListIcon, @@ -19,10 +20,11 @@ import { taskRunErrorEnhancer, } from "@trigger.dev/core/v3"; import { assertNever } from "assert-never"; -import { useEffect } from "react"; +import { type ReactNode, useEffect } from "react"; import { typedjson, useTypedFetcher } from "remix-typedjson"; import { toast } from "sonner"; import { ExitIcon } from "~/assets/icons/ExitIcon"; +import { QueuesIcon } from "~/assets/icons/QueuesIcon"; import { AdminDebugRun } from "~/components/admin/debugRun"; import { CodeBlock } from "~/components/code/CodeBlock"; import { EnvironmentCombo } from "~/components/environments/EnvironmentLabel"; @@ -82,6 +84,19 @@ import { useProject } from "~/hooks/useProject"; import { useSearchParams } from "~/hooks/useSearchParam"; import { useHasAdminAccess } from "~/hooks/useUser"; import { redirectWithErrorMessage } from "~/models/message.server"; +import { + clickhouseTimeToMs, + formatWaitMs, + QueueMetricChart, + QUEUE_METRIC_COLORS, + toNumber, + useQueueMetric, +} from "~/components/queues/QueueMetricCards"; +import { + resolveRunQueueMetrics, + type RunQueueMetrics, + type RunQueueWaiting, +} from "~/presenters/v3/RunQueueMetricsPresenter.server"; import { type Span, SpanPresenter, type SpanRun } from "~/presenters/v3/SpanPresenter.server"; import { logger } from "~/services/logger.server"; import { requireUserId } from "~/services/session.server"; @@ -91,6 +106,7 @@ import { docsPath, v3BatchPath, v3DeploymentVersionPath, + v3QueuePath, v3RunDownloadLogsPath, v3RunIdempotencyKeyResetPath, v3RunPath, @@ -144,7 +160,14 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { // `{ ...result }` collapses the union and loses the // `type === "run" | "span"` discriminant downstream in `SpanView`. if (result.type === "run") { - return typedjson({ type: "run" as const, run: result.run }); + const queueMetrics = await resolveRunQueueMetrics({ + userId, + organizationSlug, + projectParam, + envParam, + run: result.run, + }); + return typedjson({ type: "run" as const, run: result.run, queueMetrics }); } return typedjson({ type: "span" as const, span: result.span }); } catch (error) { @@ -236,6 +259,7 @@ export function SpanView({ return ( <RunBody run={fetcher.data.run} + queueMetrics={fetcher.data.queueMetrics} runParam={runParam} spanId={spanId} closePanel={closePanel} @@ -360,11 +384,13 @@ function applySpanOverrides(span: Span, spanOverrides?: SpanOverride): Span { function RunBody({ run, + queueMetrics, runParam, spanId, closePanel, }: { run: SpanRun; + queueMetrics: RunQueueMetrics | null; runParam: string; spanId: string; closePanel?: () => void; @@ -377,6 +403,12 @@ function RunBody({ const tab = value("tab"); const resetFetcher = useTypedFetcher<typeof resetIdempotencyKeyAction>(); + const queuePath = queueMetrics?.queueFriendlyId + ? v3QueuePath(organization, project, environment, { + friendlyId: queueMetrics.queueFriendlyId, + }) + : undefined; + return ( <div className="grid h-full max-h-full grid-rows-[2.5rem_2rem_1fr_minmax(3.25rem,auto)] overflow-hidden bg-background-bright"> <div className="flex items-center justify-between gap-2 overflow-x-hidden px-3 pr-2"> @@ -920,9 +952,31 @@ function RunBody({ <Property.Item> <Property.Label>Queue</Property.Label> <Property.Value> - <div>Name: {run.queue.name}</div> <div> - Concurrency key: {run.queue.concurrencyKey ? run.queue.concurrencyKey : "–"} + Name:{" "} + {queuePath ? ( + <TextLink to={queuePath}>{run.queue.name}</TextLink> + ) : ( + run.queue.name + )} + </div> + <div> + Concurrency key:{" "} + {run.queue.concurrencyKey ? ( + queuePath ? ( + <TextLink + to={`${queuePath}?view=keys&key=${encodeURIComponent( + run.queue.concurrencyKey + )}`} + > + {run.queue.concurrencyKey} + </TextLink> + ) : ( + run.queue.concurrencyKey + ) + ) : ( + "–" + )} </div> </Property.Value> </Property.Item> @@ -1064,12 +1118,26 @@ function RunBody({ </div> ) : ( <div className="flex flex-col gap-4 pt-3"> - <div className="border-b border-grid-bright pb-3"> - <SimpleTooltip - button={<TaskRunStatusCombo status={run.status} className="text-sm" />} - content={descriptionForTaskRunStatus(run.status)} + {/* The waiting-queue block carries its own Status tile, so drop the duplicate + status combo here; other runs still get it. */} + {queueMetrics?.waiting ? null : ( + <div className="border-b border-grid-bright pb-3"> + <SimpleTooltip + button={<TaskRunStatusCombo status={run.status} className="text-sm" />} + content={descriptionForTaskRunStatus(run.status)} + /> + </div> + )} + {queueMetrics?.waiting ? ( + <WaitingInQueueBlock + queueName={queueMetrics.queueName} + queuePath={queuePath} + paused={queueMetrics.paused} + waiting={queueMetrics.waiting} + status={run.status} + createdAt={run.createdAt} /> - </div> + ) : null} <RunTimeline run={run} /> {run.error && <RunError error={run.error} />} @@ -1129,6 +1197,184 @@ function RunBody({ ); } +// Trust a ClickHouse gauge only while its newest bucket is recent (10s bucket + pipeline lag); +// once it ages out, fall back to the loader's live Redis value instead of a stale count. +const LIVE_GAUGE_FRESH_MS = 90_000; + +const WAITING_STATUS_LABELS: Partial<Record<SpanRun["status"], string>> = { + PENDING: "Queued", + DELAYED: "Delayed", + PENDING_VERSION: "Pending version", +}; + +// A mini version of the queue page's stat block: same chrome, title font and alignment, smaller value. +function MiniStat({ + title, + value, + valueClassName, + suffix, + accessory, +}: { + title: ReactNode; + value: ReactNode; + valueClassName?: string; + suffix?: ReactNode; + accessory?: ReactNode; +}) { + return ( + <div className="flex min-h-[5.5rem] flex-col justify-between gap-3 rounded-sm border border-grid-dimmed bg-background-bright p-3"> + <div className="flex items-center justify-between gap-1"> + <Header3 className="leading-6">{title}</Header3> + {accessory ? <div className="-my-1 shrink-0">{accessory}</div> : null} + </div> + <div className="flex flex-wrap items-baseline gap-1"> + <span + className={cn( + "text-2xl font-normal leading-none tabular-nums text-text-bright", + valueClassName + )} + > + {value} + </span> + {suffix ? <span className="text-xs tabular-nums text-text-dimmed">{suffix}</span> : null} + </div> + </div> + ); +} + +// A compact "mini queue" for a waiting run: the queue page's stat blocks + charts, scaled down. +function WaitingInQueueBlock({ + queueName, + queuePath, + paused, + waiting, + status, + createdAt, +}: { + queueName: string; + queuePath: string | undefined; + paused: boolean; + waiting: RunQueueWaiting; + status: SpanRun["status"]; + createdAt: Date; +}) { + // Latest gauges from ClickHouse (as on the queue page), polled so the blocks keep ticking. Trust + // the newest bucket only while fresh; otherwise fall back to the loader's live values. + const { rows: liveRows } = useQueueMetric( + `SELECT timeBucket() AS t, max(max_running) AS running, max(max_queued) AS queued, max(max_limit) AS q_limit\nFROM queue_metrics\nGROUP BY t\nORDER BY t`, + { + ids: waiting.ids, + timeRange: { period: "15m", from: null, to: null }, + defaultPeriod: "15m", + queueName, + refreshIntervalMs: 15_000, + } + ); + const latest = liveRows.length > 0 ? liveRows[liveRows.length - 1] : undefined; + const latestBucketMs = latest ? clickhouseTimeToMs(latest.t) : NaN; + const fresh = + latest && Number.isFinite(latestBucketMs) && Date.now() - latestBucketMs < LIVE_GAUGE_FRESH_MS + ? latest + : undefined; + + const key = waiting.concurrencyKey; + const running = fresh ? toNumber(fresh.running) : waiting.running; + const limit = waiting.concurrencyLimit ?? (fresh ? toNumber(fresh.q_limit) || null : null); + + // The concurrency limit applies per key on keyed queues, so the tile has to compare like with + // like: the key's own running count against the per-key limit. `running` above is queue-wide, + // which would render "8 / 2 · 100%" while this run's key sits at 1 of 2. PENDING renders as + // "Queued". + const runningAgainstLimit = key ? key.running : running; + const atLimit = limit !== null && runningAgainstLimit >= limit; + const showAtLimit = status === "PENDING" && atLimit && !paused; + const pct = + limit && limit > 0 ? Math.min(100, Math.round((runningAgainstLimit / limit) * 100)) : null; + const waitedMs = Math.max(0, Date.now() - new Date(createdAt).getTime()); + + // Why the run is held, surfaced as a warning icon on the Status tile (queue-page style) rather + // than a separate sentence. + const warningMessage = paused + ? "Queue is paused. Runs will not start until it is resumed." + : showAtLimit + ? "At the concurrency limit. This run starts when a running one finishes." + : null; + + return ( + <div className="@container flex flex-col gap-3 border-b border-grid-bright pb-4"> + {/* Responsive to the side panel width (container query, not viewport): 3-up while there's + room, stacking to a single column only once the panel gets narrow. */} + <div className="grid grid-cols-1 gap-2 @[22rem]:grid-cols-3"> + <MiniStat + title={ + <span className="flex items-center gap-1.5"> + Status + {warningMessage ? ( + <SimpleTooltip + button={<ExclamationTriangleIcon className="size-4 text-warning" />} + content={warningMessage} + /> + ) : null} + </span> + } + value={WAITING_STATUS_LABELS[status] ?? "Waiting"} + valueClassName="tracking-tight" + /> + <MiniStat + title="Concurrency" + value={runningAgainstLimit.toLocaleString()} + valueClassName={cn(atLimit && "text-warning")} + suffix={ + limit !== null + ? `/ ${limit.toLocaleString()}${pct !== null ? ` · ${pct}%` : ""}` + : "of ∞" + } + /> + <MiniStat + title="Waiting for" + value={formatDurationMilliseconds(waitedMs, { style: "short", maxDecimalPoints: 0 })} + /> + </div> + + <div className="rounded-sm border border-grid-dimmed bg-background-bright p-3"> + <div className="mb-2 flex items-center justify-between gap-1.5"> + <div className="flex items-center gap-1.5"> + <Header3 className="leading-6">Scheduling delay</Header3> + <InfoIconTooltip + content="Wait from eligible to dequeued (p50/p95)." + contentClassName="max-w-xs" + /> + </div> + {queuePath ? ( + <LinkButton + variant="secondary/small-icon" + LeadingIcon={QueuesIcon} + leadingIconClassName="text-queues" + to={queuePath} + tooltip="View queue" + /> + ) : null} + </div> + <div className="h-40"> + <QueueMetricChart + query={`SELECT timeBucket() AS t,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[1]) AS p50,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + series={[ + { key: "p50", label: "p50", color: QUEUE_METRIC_COLORS.p50 }, + { key: "p95", label: "p95", color: QUEUE_METRIC_COLORS.p95 }, + ]} + ids={waiting.ids} + timeRange={{ period: "30m", from: null, to: null }} + defaultPeriod="30m" + queueName={queueName} + valueFormat={formatWaitMs} + fillGaps + /> + </div> + </div> + </div> + ); +} + // Trace export menu items: copy the trace as Markdown (for pasting into an AI // assistant) plus a download per format. The export route streams `?format=` // server-side. diff --git a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts new file mode 100644 index 00000000000..7d657a781c5 --- /dev/null +++ b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts @@ -0,0 +1,180 @@ +import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; +import { findEnvironmentById, hasAccessToEnvironment } from "~/models/runtimeEnvironment.server"; +import { requireUserId } from "~/services/session.server"; +import { canAccessQueueMetricsUi } from "~/v3/canAccessQueueMetricsUi.server"; +import { engine } from "~/v3/runEngine.server"; + +// One page of a queue's concurrency keys. The ClickHouse tier (queue_metrics_ck_v1) is the +// paginated authority — ranked by peak backlog over the window with the total on every row +// (single scan) — and the ≤PER_PAGE keys on the page are enriched with live "now" counts from +// Redis (O(page), independent of total key cardinality). This replaces the old top-50 cap. +export const CONCURRENCY_KEYS_PER_PAGE = 25; + +// Matches QUEUE_METRICS_DEFAULT_PERIOD (the detail page's TimeFilter default). +const DEFAULT_PERIOD = "1d"; + +const Body = z.object({ + organizationId: z.string(), + projectId: z.string(), + environmentId: z.string(), + /** The queue's full name (e.g. `task/my-task` or a custom queue name). */ + queueName: z.string(), + period: z.string().nullish(), + from: z.string().nullish(), + to: z.string().nullish(), + /** Case-insensitive substring filter on the key. */ + search: z.string().nullish(), + page: z.number().int().min(1).default(1), +}); + +export type ConcurrencyKeyRow = { + key: string; + queued: number; + running: number; + oldestWaitMs: number | null; + started: number; + peakBacklog: number; + peakRunning: number; + meanWaitMs: number; +}; + +export type ConcurrencyKeysResponse = + | { + success: true; + rows: ConcurrencyKeyRow[]; + total: number; + page: number; + perPage: number; + loadedAt: number; + } + | { success: false; error: string }; + +// Snap the window to a minute grid so repeated loads within a bucket produce identical query +// params and share ClickHouse query-cache entries (same trick as the queue-list ranking). +function formatClickhouseDateTime(date: Date): string { + return date.toISOString().slice(0, 19).replace("T", " "); +} +function floorToMinute(ms: number): number { + return Math.floor(ms / 60_000) * 60_000; +} +function ceilToMinute(ms: number): number { + return Math.ceil(ms / 60_000) * 60_000; +} + +export const action = async ({ request }: ActionFunctionArgs) => { + const userId = await requireUserId(request); + + const submission = Body.safeParse(await request.json()); + if (!submission.success) { + return json<ConcurrencyKeysResponse>( + { success: false, error: "Invalid input" }, + { status: 400 } + ); + } + const { organizationId, projectId, environmentId, queueName, period, from, to, search, page } = + submission.data; + + const hasAccess = await hasAccessToEnvironment({ + environmentId, + projectId, + organizationId, + userId, + }); + if (!hasAccess) { + return json<ConcurrencyKeysResponse>( + { success: false, error: "You don't have permission for this resource" }, + { status: 403 } + ); + } + + // Needed as the tenant scope for the live Redis lookup (organization/project/environment ids). + const environment = await findEnvironmentById(environmentId); + if (!environment) { + return json<ConcurrencyKeysResponse>( + { success: false, error: "Environment not found" }, + { status: 404 } + ); + } + + // Gate on the per-org Queue Metrics UI flag, matching the queue detail page and run inspector, so + // this endpoint's data isn't reachable for orgs that can't see the UI. 404 (not 403) to hide it. + if ( + !(await canAccessQueueMetricsUi({ + userId, + organizationSlug: environment.organization.slug, + })) + ) { + return json<ConcurrencyKeysResponse>({ success: false, error: "Not found" }, { status: 404 }); + } + + const range = timeFilterFromTo({ + period: period ?? undefined, + from: from ?? undefined, + to: to ?? undefined, + defaultPeriod: DEFAULT_PERIOD, + }); + const startTime = formatClickhouseDateTime(new Date(floorToMinute(range.from.getTime()))); + const endTime = formatClickhouseDateTime(new Date(ceilToMinute(range.to.getTime()))); + + try { + const clickhouse = await clickhouseFactory.getClickhouseForOrganization( + organizationId, + "queueMetrics" + ); + + const [rankingError, rankingRows] = await clickhouse.queueMetrics.concurrencyKeyRanking({ + organizationId, + projectId, + environmentId, + queueName, + startTime, + endTime, + nameContains: search?.trim() ?? "", + limit: CONCURRENCY_KEYS_PER_PAGE, + offset: (page - 1) * CONCURRENCY_KEYS_PER_PAGE, + }); + if (rankingError) { + throw rankingError; + } + + const total = rankingRows?.[0]?.ranked_total ?? 0; + const keys = (rankingRows ?? []).map((r) => r.concurrency_key); + + // Enrich just this page's keys with live "now" counts from Redis. + const live = await engine.concurrencyKeyLiveStats(environment, queueName, keys); + const loadedAt = Date.now(); + + const rows: ConcurrencyKeyRow[] = (rankingRows ?? []).map((r) => { + const l = live.get(r.concurrency_key); + const oldestWaitMs = + l && l.oldestEnqueuedAt != null ? Math.max(0, loadedAt - l.oldestEnqueuedAt) : null; + return { + key: r.concurrency_key, + queued: l?.queued ?? 0, + running: l?.running ?? 0, + oldestWaitMs, + started: r.started, + peakBacklog: r.peak_backlog, + peakRunning: r.peak_running, + meanWaitMs: r.mean_wait_ms, + }; + }); + + return json<ConcurrencyKeysResponse>({ + success: true, + rows, + total, + page, + perPage: CONCURRENCY_KEYS_PER_PAGE, + loadedAt, + }); + } catch (error) { + return json<ConcurrencyKeysResponse>( + { success: false, error: error instanceof Error ? error.message : "Query failed" }, + { status: 400 } + ); + } +}; diff --git a/apps/webapp/app/routes/storybook.charts/route.tsx b/apps/webapp/app/routes/storybook.charts/route.tsx index 8c7580cb613..040211437ad 100644 --- a/apps/webapp/app/routes/storybook.charts/route.tsx +++ b/apps/webapp/app/routes/storybook.charts/route.tsx @@ -13,6 +13,7 @@ import { useDateRange, } from "~/components/primitives/charts/DateRangeContext"; import type { ZoomRange } from "~/components/primitives/charts/hooks/useZoomSelection"; +import { MiniLineChart } from "~/components/metrics/MiniLineChart"; import { Paragraph } from "~/components/primitives/Paragraph"; import { RadioGroup, RadioGroupItem } from "~/components/primitives/RadioButton"; import SegmentedControl from "~/components/primitives/SegmentedControl"; @@ -262,6 +263,126 @@ function ChartsDashboard() { /> </Card.Content> </Card> + + {/* Line with per-bucket warning overlay (queues redesign) */} + <Card> + <Card.Header> + <div className="flex items-center gap-1.5"> + <IconTimeline className="size-5 text-warning" /> + Warning overlay{" "} + <span className="font-normal text-text-dimmed">(per-bucket recolour)</span> + </div> + </Card.Header> + <Card.Content> + {/* The base line stays the series colour; buckets strictly above `threshold` retrace + in the warning colour, so the same line reads blue -> yellow -> blue as it crosses. */} + <Chart.Root config={queueDepthConfig} data={API_DATA.queueDepthData} dataKey="day"> + <Chart.Line + lineType="monotone" + warningOverlay={{ threshold: 5000 }} + referenceLines={[{ y: 5000, label: "Threshold 5,000" }]} + xAxisProps={{ tickFormatter: xAxisTickFormatter }} + tooltipLabelFormatter={tooltipLabelFormatter} + /> + </Chart.Root> + </Card.Content> + </Card> + + {/* Line with gradient threshold split (queues redesign) */} + <Card> + <Card.Header> + <div className="flex items-center gap-1.5"> + <IconTimeline className="size-5 text-warning" /> + Threshold stroke{" "} + <span className="font-normal text-text-dimmed">(gradient split)</span> + </div> + </Card.Header> + <Card.Content> + {/* Gradient split above the threshold. Best when the threshold sits mid-domain — if it + sits far below the data max the split collapses onto the baseline; prefer + `warningOverlay` in that case (see the Warning overlay example). */} + <Chart.Root config={queueDepthConfig} data={API_DATA.queueDepthData} dataKey="day"> + <Chart.Line + lineType="monotone" + thresholdStroke={{ value: 5000, aboveColor: "var(--color-warning)" }} + xAxisProps={{ tickFormatter: xAxisTickFormatter }} + tooltipLabelFormatter={tooltipLabelFormatter} + /> + </Chart.Root> + </Card.Content> + </Card> + + {/* Line with outside-placed reference line label (queues redesign) */} + <Card> + <Card.Header> + <div className="flex items-center gap-1.5"> + <IconTimeline className="size-5 text-indigo-500" /> + Reference line{" "} + <span className="font-normal text-text-dimmed">(label outside, in the gutter)</span> + </div> + </Card.Header> + <Card.Content> + {/* labelPlacement: "outside" renders the label in the right gutter at the line's y; + the chart's right margin is widened automatically so it isn't clipped. */} + <Chart.Root config={queueDepthConfig} data={API_DATA.queueDepthData} dataKey="day"> + <Chart.Line + lineType="monotone" + referenceLines={[{ y: 8000, label: "Limit 8,000", labelPlacement: "outside" }]} + xAxisProps={{ tickFormatter: xAxisTickFormatter }} + tooltipLabelFormatter={tooltipLabelFormatter} + /> + </Chart.Root> + </Card.Content> + </Card> + + {/* MiniLineChart backlog sparklines (queues redesign) */} + <Card> + <Card.Header> + <div className="flex items-center gap-1.5"> + <IconTimeline className="size-5 text-tasks-bright" /> + Mini line chart{" "} + <span className="font-normal text-text-dimmed">(inline backlog sparkline)</span> + </div> + </Card.Header> + <Card.Content> + {/* Fixed-size inline sparkline sized for a table cell, with a trailing peak label. The + second row passes aligned `throttled` buckets so throttled stretches retrace the same + line in the warning colour. */} + <table className="w-full text-sm"> + <thead> + <tr className="border-b border-grid-dimmed text-left text-text-dimmed"> + <th className="py-1.5 pr-4 font-normal">Queue</th> + <th className="py-1.5 font-normal">Backlog</th> + </tr> + </thead> + <tbody> + <tr className="border-b border-grid-dimmed"> + <td className="py-1.5 pr-4 text-text-bright">emails</td> + <td className="py-1.5"> + <MiniLineChart + data={API_DATA.miniLineData} + peak={Math.max(...API_DATA.miniLineData)} + unitLabel={{ singular: "queued", plural: "queued" }} + color="var(--color-tasks)" + /> + </td> + </tr> + <tr> + <td className="py-1.5 pr-4 text-text-bright">image-processing</td> + <td className="py-1.5"> + <MiniLineChart + data={API_DATA.miniLineThrottledData} + throttled={API_DATA.miniLineThrottledBuckets} + peak={Math.max(...API_DATA.miniLineThrottledData)} + unitLabel={{ singular: "queued", plural: "queued" }} + color="var(--color-tasks)" + /> + </td> + </tr> + </tbody> + </table> + </Card.Content> + </Card> </div> </div> ); @@ -681,6 +802,30 @@ const API_DATA = { "analyze-document": 5678, }, ], + // Single-series queue depth that crosses the 5,000 threshold twice (blue -> yellow -> blue), + // used by the warning overlay, threshold stroke and outside reference-line examples. + queueDepthData: [ + { day: "2023-11-01", queued: 1200 }, + { day: "2023-11-02", queued: 2400 }, + { day: "2023-11-03", queued: 3800 }, + { day: "2023-11-04", queued: 4600 }, + { day: "2023-11-05", queued: 6200 }, + { day: "2023-11-06", queued: 7400 }, + { day: "2023-11-07", queued: 6800 }, + { day: "2023-11-08", queued: 5200 }, + { day: "2023-11-09", queued: 3600 }, + { day: "2023-11-10", queued: 2800 }, + { day: "2023-11-11", queued: 3400 }, + { day: "2023-11-12", queued: 5600 }, + { day: "2023-11-13", queued: 7800 }, + { day: "2023-11-14", queued: 6400 }, + { day: "2023-11-15", queued: 4200 }, + ], + // Inline sparkline buckets (oldest first). No throttling on this one. + miniLineData: [3, 5, 8, 12, 9, 14, 20, 18, 11, 7, 4, 6], + // A backlog that was throttled across a stretch in the middle; `throttled` aligns 1:1 with `data`. + miniLineThrottledData: [2, 4, 9, 16, 24, 30, 27, 19, 12, 8, 5, 3], + miniLineThrottledBuckets: [0, 0, 0, 6, 14, 18, 11, 0, 0, 0, 0, 0], }; const lineChartConfig = { @@ -694,6 +839,13 @@ const lineChartConfig = { }, } satisfies ChartConfig; +const queueDepthConfig = { + queued: { + label: "Queue depth", + color: "var(--color-tasks)", + }, +} satisfies ChartConfig; + const barChartBigDatasetConfig = { "sync-data": { label: ( diff --git a/apps/webapp/app/routes/storybook.layout/route.tsx b/apps/webapp/app/routes/storybook.layout/route.tsx new file mode 100644 index 00000000000..08242fe51da --- /dev/null +++ b/apps/webapp/app/routes/storybook.layout/route.tsx @@ -0,0 +1,373 @@ +import { useState } from "react"; +import { PageContainer } from "~/components/layout/AppLayout"; +import { MetricsLayout } from "~/components/layout/MetricsLayout"; +import { Badge } from "~/components/primitives/Badge"; +import { Header3 } from "~/components/primitives/Headers"; +import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import SegmentedControl from "~/components/primitives/SegmentedControl"; +import { + Table, + TableBody, + TableCell, + TableHeader, + TableHeaderCell, + TableRow, +} from "~/components/primitives/Table"; +import { TabButton, TabContainer } from "~/components/primitives/Tabs"; +import { cn } from "~/utils/cn"; + +// A placeholder for a search input / TimeFilter / pagination — the real pages drop live controls +// into the Filters slot; here we only show the row shape. +function FilterChip({ children }: { children: React.ReactNode }) { + return ( + <div className="flex h-8 items-center gap-1.5 rounded-sm border border-grid-dimmed bg-background-bright px-3 text-sm text-text-dimmed"> + {children} + </div> + ); +} + +// A stat BigNumber-style tile. +function StatTile({ label, value }: { label: string; value: string }) { + return ( + <div className="rounded-sm border border-grid-dimmed bg-background-bright p-4"> + <Header3 className="leading-6">{label}</Header3> + <div className="mt-2 text-[3rem] font-normal leading-none tabular-nums text-text-bright"> + {value} + </div> + </div> + ); +} + +// A chart-card-style tile with a fake bar sparkline so the grid's height + column behaviour is +// visible. +function ChartTile({ label, className }: { label: string; className?: string }) { + return ( + <div + className={cn( + "flex flex-col rounded-sm border border-grid-dimmed bg-background-bright p-3", + className + )} + > + <Paragraph variant="small/bright">{label}</Paragraph> + <div className="mt-3 flex min-h-0 flex-1 items-end gap-px"> + {Array.from({ length: 40 }).map((_, i) => ( + <div + key={i} + className="flex-1 rounded-t-[1px] bg-primary/60" + style={{ height: `${20 + Math.abs(Math.sin(i / 3)) * 70}%` }} + /> + ))} + </div> + </div> + ); +} + +function PlaceholderTable() { + return ( + <Table containerClassName="border-t"> + <TableHeader> + <TableRow> + <TableHeaderCell>Name</TableHeaderCell> + <TableHeaderCell alignment="right">Queued</TableHeaderCell> + <TableHeaderCell alignment="right">Running</TableHeaderCell> + <TableHeaderCell alignment="right">Limit</TableHeaderCell> + </TableRow> + </TableHeader> + <TableBody> + {[ + { name: "background", queued: 0, running: 4, limit: 19 }, + { name: "per-tenant", queued: 83, running: 11, limit: 10 }, + { name: "emails", queued: 2, running: 1, limit: 50 }, + ].map((row) => ( + <TableRow key={row.name}> + <TableCell>{row.name}</TableCell> + <TableCell alignment="right">{row.queued}</TableCell> + <TableCell alignment="right">{row.running}</TableCell> + <TableCell alignment="right">{row.limit}</TableCell> + </TableRow> + ))} + </TableBody> + </Table> + ); +} + +// A tall filler so a scroll region visibly overflows its container. +function ScrollFiller({ label, rows }: { label: string; rows: number }) { + return ( + <div className="flex flex-col gap-2 p-3"> + {Array.from({ length: rows }).map((_, i) => ( + <div + key={i} + className="flex items-center justify-between rounded-sm border border-grid-dimmed bg-background-bright px-3 py-2 text-sm text-text-dimmed" + > + <span> + {label} row {i + 1} + </span> + <span className="tabular-nums">{Math.round(Math.abs(Math.sin(i)) * 1000)}</span> + </div> + ))} + </div> + ); +} + +// The config panel dropped into MetricsLayout.Sidebar in the sidebar demos. +function SidebarPanel({ resizable }: { resizable?: boolean }) { + return ( + <div className="flex h-full flex-col border-l border-grid-dimmed bg-background-bright"> + <div className="flex h-10 shrink-0 items-center border-b border-grid-dimmed px-3"> + <Header3>Sidebar slot</Header3> + </div> + <div className="flex flex-col gap-3 overflow-y-auto p-3"> + <Paragraph variant="small"> + {resizable + ? "This sidebar is resizable — drag the handle on its left edge. Pass an autosaveId + a loader snapshot to persist the split across reloads." + : "This sidebar is fixed-width (width prop). The main column fills the rest and owns the page scroll."} + </Paragraph> + <Badge variant="extra-small">{resizable ? "resizable" : "fixed width"}</Badge> + {Array.from({ length: 6 }).map((_, i) => ( + <FilterChip key={i}>Config option {i + 1}</FilterChip> + ))} + </div> + </div> + ); +} + +// The overview demo. Every slot carries its baked chrome — no className is passed to Root, +// Filters, Grid or Content. The pinned Filters bar spreads a left and right cluster; the Grids +// bake the page gutter and adapt their columns (or take an explicit `columns` / `kind="charts"`); +// Content toggles between a full-bleed table and an inset panel. The whole page scrolls as one. +function OverviewDemo() { + const [tab, setTab] = useState<"panel" | "table">("panel"); + + return ( + <MetricsLayout.Root> + {/* Filters slot — the pinned bar directly under the NavBar. Left + right clusters as child + divs; the slot bakes the 40px height, border and insets. */} + <MetricsLayout.Filters> + <div className="flex items-center gap-2"> + <FilterChip>Search…</FilterChip> + <FilterChip>Period: 7d</FilterChip> + <Badge variant="extra-small">Filters slot</Badge> + </div> + <FilterChip>{"< 1 / 4 >"}</FilterChip> + </MetricsLayout.Filters> + + {/* Grid slot — 4 stat tiles. Columns are derived from the tile count: two-up, four-up + from lg. The gutter + gap are baked. */} + <div className="px-3 text-xs uppercase text-text-dimmed"> + Grid — 4 tiles (auto: 2-up, 4-up from lg) + </div> + <MetricsLayout.Grid> + <StatTile label="Queued" value="83" /> + <StatTile label="Running" value="15" /> + <StatTile label="Allocated" value="29" /> + <StatTile label="Limit" value="25" /> + </MetricsLayout.Grid> + + {/* Grid slot — kind="charts" bakes the fixed chart-row height (no wrapper needed). */} + <div className="px-3 text-xs uppercase text-text-dimmed"> + Grid — kind="charts" (fixed row height, auto columns) + </div> + <MetricsLayout.Grid kind="charts"> + <ChartTile label="Env saturation" /> + <ChartTile label="Backlog" /> + <ChartTile label="Scheduling delay p95" /> + <ChartTile label="Throttled" /> + </MetricsLayout.Grid> + + {/* Grid slot — 3 tiles. The same component lays out one-up, three-up from sm, proving the + grid adapts to the child count. */} + <div className="px-3 text-xs uppercase text-text-dimmed"> + Grid — 3 tiles (auto: 1-up, 3-up from sm) + </div> + <MetricsLayout.Grid> + <StatTile label="Concurrency" value="11" /> + <StatTile label="Queued" value="83" /> + <StatTile label="Oldest wait" value="34m" /> + </MetricsLayout.Grid> + + {/* Grid slot — explicit columns for a chart grid that should always be two-up regardless + of tile count. */} + <div className="px-3 text-xs uppercase text-text-dimmed"> + Grid — explicit columns={{ base: 1, sm: 2 }} (5 tiles) + </div> + <MetricsLayout.Grid columns={{ base: 1, sm: 2 }}> + <ChartTile label="Concurrency" className="aspect-[2/1]" /> + <ChartTile label="Queue depth" className="aspect-[2/1]" /> + <ChartTile label="Throughput" className="aspect-[2/1]" /> + <ChartTile label="Scheduling delay" className="aspect-[2/1]" /> + <ChartTile label="Throttled" className="aspect-[2/1]" /> + </MetricsLayout.Grid> + + {/* Content slot — a doubled separation above it is baked in, so the tiles read as their own + band. `inset` toggles between a padded column (panel) and full-bleed (edge-to-edge + table). */} + <MetricsLayout.Content inset={tab === "panel"}> + <TabContainer> + <TabButton + isActive={tab === "panel"} + layoutId="layout-story" + onClick={() => setTab("panel")} + > + Panel (inset) + </TabButton> + <TabButton + isActive={tab === "table"} + layoutId="layout-story" + onClick={() => setTab("table")} + > + Table (full-bleed) + </TabButton> + </TabContainer> + {tab === "table" ? ( + <PlaceholderTable /> + ) : ( + <div className="rounded-sm border border-grid-dimmed bg-background-bright p-3"> + <Paragraph variant="small"> + With <code>inset</code>, Content becomes a padded column: this panel and the tabs + above it sit on the standard page gutter. Switch to the table to see Content go + full-bleed — the table spans edge to edge with its own top border. Either way the + whole page (filters aside) shares one vertical scroll. + </Paragraph> + </div> + )} + </MetricsLayout.Content> + </MetricsLayout.Root> + ); +} + +// Fixed-width sidebar: a <MetricsLayout.Sidebar> child flips Root into a [main | sidebar] layout. +// The main column keeps its normal top-to-bottom slots and owns the page scroll. +function SidebarFixedDemo() { + return ( + <MetricsLayout.Root> + <MetricsLayout.Filters> + <div className="flex items-center gap-2"> + <FilterChip>Search…</FilterChip> + <Badge variant="extra-small">main column</Badge> + </div> + </MetricsLayout.Filters> + <MetricsLayout.Grid> + <StatTile label="Queued" value="83" /> + <StatTile label="Running" value="15" /> + <StatTile label="Allocated" value="29" /> + </MetricsLayout.Grid> + <MetricsLayout.Content> + <PlaceholderTable /> + </MetricsLayout.Content> + + <MetricsLayout.Sidebar width="320px"> + <SidebarPanel /> + </MetricsLayout.Sidebar> + </MetricsLayout.Root> + ); +} + +// Resizable sidebar: same [main | sidebar] layout, but the split is draggable via the shared +// Resizable primitives. autosaveId persists the split to a cookie (in a real page the loader +// hydrates it back through a snapshot); here it persists live within the session. +function SidebarResizableDemo() { + return ( + <MetricsLayout.Root> + <MetricsLayout.Filters> + <div className="flex items-center gap-2"> + <FilterChip>Search…</FilterChip> + <Badge variant="extra-small">drag the handle →</Badge> + </div> + </MetricsLayout.Filters> + <MetricsLayout.Grid> + <StatTile label="Queued" value="83" /> + <StatTile label="Running" value="15" /> + <StatTile label="Allocated" value="29" /> + </MetricsLayout.Grid> + <MetricsLayout.Content> + <PlaceholderTable /> + </MetricsLayout.Content> + + <MetricsLayout.Sidebar + resizable + autosaveId="storybook-metrics-sidebar" + min="260px" + defaultSize="360px" + max="520px" + > + <SidebarPanel resizable /> + </MetricsLayout.Sidebar> + </MetricsLayout.Root> + ); +} + +// scroll="regions": Root does NOT create the page scroll. It only bounds the height as a flex +// column, so the page composes its own independently-scrolling areas — here a fixed toolbar over +// two side-by-side lists that each scroll on their own. +function RegionsDemo() { + return ( + <MetricsLayout.Root scroll="regions"> + <div className="flex h-10 shrink-0 items-center gap-2 border-t border-grid-dimmed px-3"> + <Badge variant="extra-small">fixed toolbar (does not scroll)</Badge> + <FilterChip>Period: 7d</FilterChip> + </div> + <div className="grid min-h-0 flex-1 grid-cols-2 divide-x divide-grid-dimmed"> + <div className="min-h-0 overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-surface-control"> + <div className="sticky top-0 z-1 border-b border-grid-dimmed bg-background px-3 py-1.5 text-xs uppercase text-text-dimmed"> + Left region — scrolls independently + </div> + <ScrollFiller label="Left" rows={60} /> + </div> + <div className="min-h-0 overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-surface-control"> + <div className="sticky top-0 z-1 border-b border-grid-dimmed bg-background px-3 py-1.5 text-xs uppercase text-text-dimmed"> + Right region — scrolls independently + </div> + <ScrollFiller label="Right" rows={60} /> + </div> + </div> + </MetricsLayout.Root> + ); +} + +type Demo = "overview" | "sidebar-fixed" | "sidebar-resizable" | "regions"; + +const DEMO_OPTIONS: { label: string; value: Demo }[] = [ + { label: "Overview", value: "overview" }, + { label: "Sidebar (fixed)", value: "sidebar-fixed" }, + { label: "Sidebar (resizable)", value: "sidebar-resizable" }, + { label: "Scroll: regions", value: "regions" }, +]; + +/** + * Storybook for the MetricsLayout compound. A segmented control swaps between demos, each filling + * the page: + * - Overview — the base Filters / count-adaptive Grid / Content slots, page scroll. + * - Sidebar (fixed) — a fixed-width MetricsLayout.Sidebar beside the main column. + * - Sidebar (resizable) — the same, but with a draggable split (autosaveId persistence). + * - Scroll: regions — scroll="regions" with two independently-scrolling areas. + */ +export default function Story() { + const [demo, setDemo] = useState<Demo>("overview"); + + return ( + <PageContainer> + <NavBar> + <PageTitle title="MetricsLayout" /> + <PageAccessories> + <SegmentedControl + name="metrics-layout-demo" + value={demo} + options={DEMO_OPTIONS} + onChange={(value) => setDemo(value as Demo)} + /> + </PageAccessories> + </NavBar> + {demo === "overview" ? ( + <OverviewDemo /> + ) : demo === "sidebar-fixed" ? ( + <SidebarFixedDemo /> + ) : demo === "sidebar-resizable" ? ( + <SidebarResizableDemo /> + ) : ( + <RegionsDemo /> + )} + </PageContainer> + ); +} diff --git a/apps/webapp/app/routes/storybook/route.tsx b/apps/webapp/app/routes/storybook/route.tsx index 012d47827de..8309fee5b2b 100644 --- a/apps/webapp/app/routes/storybook/route.tsx +++ b/apps/webapp/app/routes/storybook/route.tsx @@ -71,6 +71,10 @@ const stories: Story[] = [ name: "Inline code", slug: "inline-code", }, + { + name: "Layout", + slug: "layout", + }, { name: "Loading bar divider", slug: "loading-bar-divider", diff --git a/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts index c471ce30d8f..7624810efe9 100644 --- a/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts +++ b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts @@ -281,6 +281,60 @@ function initializeRunsListClickhouseClient(): ClickHouse { }); } +/** + * Queue metrics (`QUEUE_METRICS_CLICKHOUSE_URL`), a mixed read+write client: the ingestion + * consumer inserts through it and every queue-metrics read goes through it. When the URL is + * unset it reproduces the previous split exactly, writing to `CLICKHOUSE_URL` and reading from + * the query pool, so the dedicated service is opt-in per deployment. + */ +const defaultQueueMetricsClickhouseClient = singleton( + "queueMetricsClickhouseClient", + initializeQueueMetricsClickhouseClient +); + +function initializeQueueMetricsClickhouseClient(): ClickHouse { + const dedicated = env.QUEUE_METRICS_CLICKHOUSE_URL; + + const writerUrl = new URL(dedicated ?? env.CLICKHOUSE_URL); + writerUrl.searchParams.delete("secure"); + + const readerUrl = new URL( + env.QUEUE_METRICS_CLICKHOUSE_READER_URL ?? + dedicated ?? + env.QUERY_CLICKHOUSE_URL ?? + env.CLICKHOUSE_URL + ); + readerUrl.searchParams.delete("secure"); + + const commonConfig = { + keepAlive: { + enabled: env.QUEUE_METRICS_CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.QUEUE_METRICS_CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.QUEUE_METRICS_CLICKHOUSE_LOG_LEVEL, + compression: { + request: env.QUEUE_METRICS_CLICKHOUSE_COMPRESSION_REQUEST === "1", + }, + maxOpenConnections: env.QUEUE_METRICS_CLICKHOUSE_MAX_OPEN_CONNECTIONS, + }; + + if (readerUrl.toString() !== writerUrl.toString()) { + return new ClickHouse({ + ...commonConfig, + writerName: "queue-metrics-writer", + writerUrl: writerUrl.toString(), + readerName: "queue-metrics-reader", + readerUrl: readerUrl.toString(), + }); + } + + return new ClickHouse({ + ...commonConfig, + name: "queue-metrics-clickhouse", + url: writerUrl.toString(), + }); +} + /** Task events (`EVENTS_CLICKHOUSE_URL`); not exported — accessed via factory. */ const defaultEventsClickhouseClient = singleton( "eventsClickhouseClient", @@ -349,7 +403,8 @@ export type ClientType = | "admin" | "engine" | "realtime" - | "runsList"; + | "runsList" + | "queueMetrics"; function buildOrgClickhouseClient(url: string, clientType: ClientType): ClickHouse { const parsed = new URL(url); @@ -422,6 +477,20 @@ function buildOrgClickhouseClient(url: string, clientType: ClientType): ClickHou }, maxOpenConnections: env.RUN_ENGINE_CLICKHOUSE_MAX_OPEN_CONNECTIONS, }); + case "queueMetrics": + return new ClickHouse({ + url: parsed.toString(), + name, + keepAlive: { + enabled: env.QUEUE_METRICS_CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.QUEUE_METRICS_CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.QUEUE_METRICS_CLICKHOUSE_LOG_LEVEL, + compression: { + request: env.QUEUE_METRICS_CLICKHOUSE_COMPRESSION_REQUEST === "1", + }, + maxOpenConnections: env.QUEUE_METRICS_CLICKHOUSE_MAX_OPEN_CONNECTIONS, + }); case "realtime": return new ClickHouse({ url: parsed.toString(), @@ -509,6 +578,8 @@ export class ClickhouseFactory { return defaultRealtimeClickhouseClient; case "runsList": return defaultRunsListClickhouseClient; + case "queueMetrics": + return defaultQueueMetricsClickhouseClient; } } @@ -582,6 +653,11 @@ export function getDefaultLogsClickhouseClient(): ClickHouse { return defaultLogsClickhouseClient; } +/** Queue-metrics client for callers with no organization in scope (the ingestion consumer). */ +export function getQueueMetricsClickhouseClient(): ClickHouse { + return defaultQueueMetricsClickhouseClient; +} + // --------------------------------------------------------------------------- // Private helpers // --------------------------------------------------------------------------- diff --git a/apps/webapp/app/services/queryService.server.ts b/apps/webapp/app/services/queryService.server.ts index 57b877ed876..3a34f82cf1d 100644 --- a/apps/webapp/app/services/queryService.server.ts +++ b/apps/webapp/app/services/queryService.server.ts @@ -7,11 +7,17 @@ import { type TSQLQueryResult, } from "@internal/clickhouse"; import type { CustomerQuerySource } from "@trigger.dev/database"; -import type { TableSchema, WhereClauseCondition } from "@internal/tsql"; +import { + calculateTimeBucketInterval, + type TableSchema, + type TimeBucketInterval, + type WhereClauseCondition, +} from "@internal/tsql"; import { z } from "zod"; import { prisma } from "~/db.server"; import { env } from "~/env.server"; import { clickhouseFactory } from "./clickhouse/clickhouseFactoryInstance.server"; +import type { ClientType } from "./clickhouse/clickhouseFactory.server"; import { queryConcurrencyLimiter, DEFAULT_ORG_CONCURRENCY_LIMIT, @@ -110,6 +116,66 @@ export type ExecuteQueryResult<T> = } | { success: false; error: Error }; +/** Own-property flag tagged on the transient "query concurrency exceeded" rejection (retryable). */ +const QUERY_CONCURRENCY_REJECTION_FLAG = "__queryConcurrencyRejection"; + +/** True for the transient concurrency-limit rejection — a stable signal callers can retry on. */ +export function isQueryConcurrencyRejection(error: unknown): boolean { + return ( + typeof error === "object" && + error !== null && + (error as Record<string, unknown>)[QUERY_CONCURRENCY_REJECTION_FLAG] === true + ); +} + +const INTERVAL_UNIT_SECONDS: Record<TimeBucketInterval["unit"], number> = { + SECOND: 1, + MINUTE: 60, + HOUR: 3_600, + DAY: 86_400, + WEEK: 604_800, + MONTH: 2_592_000, +}; + +function floorToSeconds(date: Date, alignSeconds: number): Date { + const ms = alignSeconds * 1000; + return new Date(Math.floor(date.getTime() / ms) * ms); +} + +/** + * ClickHouse client a table's reads run on. A table can name its own pool (`queryClient`) so a + * heavy read family lands on its own service; everything else shares the query pool. + */ +function resolveQueryClientType(schema: TableSchema | undefined): ClientType { + switch (schema?.queryClient) { + case "queueMetrics": + return "queueMetrics"; + default: + return "query"; + } +} + +/** + * Swap a table for one of its rollups when the query's bucket interval is at least the + * rollup's granularity. The rollup has identical logical columns, so only the physical + * table (and therefore rows read) changes. + */ +function resolveRollup(schema: TableSchema, timeRange: { from: Date; to: Date }): TableSchema { + if (!schema.rollups || schema.rollups.length === 0) { + return schema; + } + const interval = calculateTimeBucketInterval( + timeRange.from, + timeRange.to, + schema.timeBucketThresholds + ); + const intervalSeconds = interval.value * INTERVAL_UNIT_SECONDS[interval.unit]; + const best = [...schema.rollups] + .sort((a, b) => b.minIntervalSeconds - a.minIntervalSeconds) + .find((r) => r.minIntervalSeconds <= intervalSeconds); + return best ? { ...schema, clickhouseName: best.clickhouseName } : schema; +} + export async function getDefaultPeriod(organizationId: string): Promise<string> { const idealDefaultPeriodDays = 7; const maxQueryPeriod = await getLimit(organizationId, "queryPeriodDays", 30); @@ -164,7 +230,11 @@ export async function executeQuery<TOut extends z.ZodSchema>( acquireResult.reason === "key_limit" ? `You've exceeded your query concurrency of ${orgLimit} for this project. Please try again later.` : "We're experiencing a lot of queries at the moment. Please try again later."; - return { success: false, error: new QueryError(errorMessage, { query: options.query }) }; + const error = new QueryError(errorMessage, { query: options.query }); + // Stable marker so callers can retry on a transient concurrency rejection without + // matching the message text (which is free to change). + Object.assign(error, { [QUERY_CONCURRENCY_REJECTION_FLAG]: true }); + return { success: false, error }; } // Detect which table the query targets to determine the time column @@ -183,6 +253,14 @@ export async function executeQuery<TOut extends z.ZodSchema>( defaultPeriod, }); + // Align the time bounds so repeated auto-refresh queries produce identical query + // params and can share ClickHouse query-cache entries (params are part of the key). + const alignSeconds = matchedSchema?.queryCache?.alignSeconds; + if (alignSeconds) { + if (timeFilter.from) timeFilter.from = floorToSeconds(timeFilter.from, alignSeconds); + if (timeFilter.to) timeFilter.to = floorToSeconds(timeFilter.to, alignSeconds); + } + // Calculate the effective "from" date the user is requesting (for period clipping check) // This is null only when the user specifies just a "to" date (rare case) let requestedFromDate: Date | null = null; @@ -192,6 +270,9 @@ export async function executeQuery<TOut extends z.ZodSchema>( // Period specified (or default) - calculate from now const periodMs = parse(timeFilter.period ?? defaultPeriod) ?? 7 * 24 * 60 * 60 * 1000; requestedFromDate = new Date(Date.now() - periodMs); + if (alignSeconds) { + requestedFromDate = floorToSeconds(requestedFromDate, alignSeconds); + } } // Build the fallback WHERE condition based on what the user specified @@ -207,7 +288,10 @@ export async function executeQuery<TOut extends z.ZodSchema>( } const maxQueryPeriod = await getLimit(organizationId, "queryPeriodDays", 30); - const maxQueryPeriodDate = new Date(Date.now() - maxQueryPeriod * 24 * 60 * 60 * 1000); + let maxQueryPeriodDate = new Date(Date.now() - maxQueryPeriod * 24 * 60 * 60 * 1000); + if (alignSeconds) { + maxQueryPeriodDate = floorToSeconds(maxQueryPeriodDate, alignSeconds); + } // Check if the requested time period exceeds the plan limit const periodClipped = requestedFromDate !== null && requestedFromDate < maxQueryPeriodDate; @@ -255,6 +339,10 @@ export async function executeQuery<TOut extends z.ZodSchema>( to: to ?? undefined, defaultPeriod, }); + if (alignSeconds) { + timeRange.from = floorToSeconds(timeRange.from, alignSeconds); + timeRange.to = floorToSeconds(timeRange.to, alignSeconds); + } try { // Build field mappings for project_ref → project_id and environment_id → slug translation @@ -275,12 +363,21 @@ export async function executeQuery<TOut extends z.ZodSchema>( const queryClickhouse = await clickhouseFactory.getClickhouseForOrganization( organizationId, - "query" + resolveQueryClientType(matchedSchema) ); + // Serve coarse-bucket queries from the table's rollup when one qualifies. + const effectiveSchemas = matchedSchema?.rollups + ? querySchemas.map((s) => (s === matchedSchema ? resolveRollup(s, timeRange) : s)) + : querySchemas; + + const queryCacheSettings: ClickHouseSettings = matchedSchema?.queryCache + ? { use_query_cache: 1, query_cache_ttl: matchedSchema.queryCache.ttlSeconds } + : {}; + const result = await executeTSQL(queryClickhouse.reader, { ...baseOptions, schema: z.record(z.any()), - tableSchema: querySchemas, + tableSchema: effectiveSchemas, transformValues: true, enforcedWhereClause, fieldMappings, @@ -290,6 +387,7 @@ export async function executeQuery<TOut extends z.ZodSchema>( timeRange, clickhouseSettings: { ...getDefaultClickhouseSettings(), + ...queryCacheSettings, ...baseOptions.clickhouseSettings, // Allow caller overrides if needed }, querySettings: { diff --git a/apps/webapp/app/utils/pathBuilder.ts b/apps/webapp/app/utils/pathBuilder.ts index 187bc50b549..edd65f8bde4 100644 --- a/apps/webapp/app/utils/pathBuilder.ts +++ b/apps/webapp/app/utils/pathBuilder.ts @@ -522,6 +522,15 @@ export function v3QueuesPath( return `${v3EnvironmentPath(organization, project, environment)}/queues`; } +export function v3QueuePath( + organization: OrgForPath, + project: ProjectForPath, + environment: EnvironmentForPath, + queue: { friendlyId: string } +) { + return `${v3QueuesPath(organization, project, environment)}/${queue.friendlyId}`; +} + export function v3WaitpointTokensPath( organization: OrgForPath, project: ProjectForPath, diff --git a/apps/webapp/app/v3/canAccessQueueMetricsUi.server.ts b/apps/webapp/app/v3/canAccessQueueMetricsUi.server.ts new file mode 100644 index 00000000000..0e3c142b272 --- /dev/null +++ b/apps/webapp/app/v3/canAccessQueueMetricsUi.server.ts @@ -0,0 +1,26 @@ +import { prisma } from "~/db.server"; +import { FEATURE_FLAG } from "~/v3/featureFlags"; +import { makeFlag } from "~/v3/featureFlags.server"; + +// Per-org gate for the Queue Metrics dashboard UI. Org override wins over the global +// FeatureFlag table value, which wins over the off-by-default. Ingestion/emission is a +// separate global flag; this only decides whether an org sees the metrics view. +export async function canAccessQueueMetricsUi(options: { + userId: string; + organizationSlug: string; +}): Promise<boolean> { + const org = await prisma.organization.findFirst({ + where: { + slug: options.organizationSlug, + members: { some: { userId: options.userId } }, + }, + select: { featureFlags: true }, + }); + + const flag = makeFlag(); + return flag({ + key: FEATURE_FLAG.queueMetricsUiEnabled, + defaultValue: false, + overrides: (org?.featureFlags as Record<string, unknown>) ?? {}, + }); +} diff --git a/apps/webapp/app/v3/featureFlags.ts b/apps/webapp/app/v3/featureFlags.ts index 22e5541d561..78a919b20ba 100644 --- a/apps/webapp/app/v3/featureFlags.ts +++ b/apps/webapp/app/v3/featureFlags.ts @@ -23,6 +23,7 @@ export const FEATURE_FLAG = { // Grace-linger stamp carried alongside runOpsMintKind on flip. See mintFlipGrace.ts. runOpsMintKindPrev: "runOpsMintKindPrev", runOpsMintKindFlippedAt: "runOpsMintKindFlippedAt", + queueMetricsUiEnabled: "queueMetricsUiEnabled", } as const; export const FeatureFlagCatalog = { @@ -68,6 +69,9 @@ export const FeatureFlagCatalog = { // by stampMintKindFlip on a genuine flip. Display-only (see ORG_LOCKED_FLAGS). [FEATURE_FLAG.runOpsMintKindPrev]: z.enum(["cuid", "runOpsId"]), [FEATURE_FLAG.runOpsMintKindFlippedAt]: z.string().datetime(), + // Per-org access to the Queue Metrics dashboard UI (view only; emission is global and + // separate). Off unless enabled for the org. + [FEATURE_FLAG.queueMetricsUiEnabled]: z.coerce.boolean(), }; export type FeatureFlagKey = keyof typeof FeatureFlagCatalog; diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index a860d411a37..d7637f52d63 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -610,8 +610,335 @@ export const metricsSchema: TableSchema = { }; /** - * All available schemas for the query editor + * Schema definition for the queue_metrics table (trigger_dev.queue_metrics_v1). + * Pre-aggregated into 10-second buckets. Counter columns re-aggregate with sum(), + * gauges with max(), and wait_quantiles with quantilesMerge() — never FINAL. */ +export const queueMetricsSchema: TableSchema = { + name: "queue_metrics", + clickhouseName: "trigger_dev.queue_metrics_v1", + description: "Per-queue depth, concurrency, throttling, and scheduling-delay metrics", + timeConstraint: "bucket_start", + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + columns: { + environment: { + name: "environment", + clickhouseName: "environment_id", + ...column("String", { description: "The environment slug", example: "prod" }), + fieldMapping: "environment", + customRenderType: "environment", + }, + project: { + name: "project", + clickhouseName: "project_id", + ...column("String", { + description: "The project reference, they always start with `proj_`.", + example: "proj_howcnaxbfxdmwmxazktx", + }), + fieldMapping: "project", + customRenderType: "project", + }, + queue: { + name: "queue", + clickhouseName: "queue_name", + ...column("LowCardinality(String)", { + description: "The queue name", + example: "my-queue", + coreColumn: true, + }), + }, + bucket_start: { + name: "bucket_start", + ...column("DateTime", { + description: "The start of the 10-second aggregation bucket", + example: "2024-01-15 09:30:00", + coreColumn: true, + }), + }, + // Cumulative-counter delta states. Read with deltaSumTimestampMerge(<col>) (loss-tolerant, + // reset-safe), never sum(); opaque like wait_quantiles. Merging across queues is + // invalid (mixes unrelated odometers): totals must GROUP BY queue, then sum outside. + enqueue_delta: { + name: "enqueue_delta", + mergeGroupKey: "queue", + ...column("String", { + description: + "Runs enqueued (cumulative-counter delta). Read with deltaSumTimestampMerge(enqueue_delta) grouped by queue. For totals across queues, sum the per-queue results in an outer query, never merge across queues. Per-bucket values can undercount by one inter-reading delta at bucket boundaries (the bridge lives in the prior bucket's state); totals over the whole range are exact.", + }), + groupable: false, + sortable: false, + filterable: false, + }, + started_delta: { + name: "started_delta", + mergeGroupKey: "queue", + ...column("String", { + description: + "Runs dequeued/started (throughput). Read with deltaSumTimestampMerge(started_delta) grouped by queue. For totals across queues, sum the per-queue results in an outer query, never merge across queues. Per-bucket values can undercount by one inter-reading delta at bucket boundaries (the bridge lives in the prior bucket's state); totals over the whole range are exact.", + coreColumn: true, + }), + groupable: false, + sortable: false, + filterable: false, + }, + ack_delta: { + name: "ack_delta", + mergeGroupKey: "queue", + ...column("String", { + description: + "Runs acked (completed). Read with deltaSumTimestampMerge(ack_delta) grouped by queue; sum per-queue results for totals.", + }), + groupable: false, + sortable: false, + filterable: false, + }, + nack_delta: { + name: "nack_delta", + mergeGroupKey: "queue", + ...column("String", { + description: + "Runs nacked. Read with deltaSumTimestampMerge(nack_delta) grouped by queue; sum per-queue results for totals.", + }), + groupable: false, + sortable: false, + filterable: false, + }, + dlq_delta: { + name: "dlq_delta", + mergeGroupKey: "queue", + ...column("String", { + description: + "Runs dead-lettered. Read with deltaSumTimestampMerge(dlq_delta) grouped by queue; sum per-queue results for totals.", + }), + groupable: false, + sortable: false, + filterable: false, + }, + throttled_count: { + name: "throttled_count", + ...column("UInt64", { + description: "Gauge emissions where running>=limit and queued>0. Aggregate with sum().", + coreColumn: true, + }), + }, + max_queued: { + name: "max_queued", + ...column("UInt32", { + description: "Peak queue depth in the bucket. Aggregate with max().", + coreColumn: true, + fillMode: "carry", + }), + }, + max_running: { + name: "max_running", + ...column("UInt32", { + description: "Peak running (concurrency) in the bucket. Aggregate with max().", + coreColumn: true, + fillMode: "carry", + }), + }, + max_limit: { + name: "max_limit", + ...column("UInt32", { + description: "The queue concurrency limit. Aggregate with max().", + coreColumn: true, + fillMode: "carry", + }), + }, + max_env_queued: { + name: "max_env_queued", + ...column("UInt32", { + description: "Peak environment-wide queued in the bucket. Aggregate with max().", + fillMode: "carry", + }), + }, + max_env_running: { + name: "max_env_running", + ...column("UInt32", { + description: "Peak environment-wide running in the bucket. Aggregate with max().", + fillMode: "carry", + }), + }, + max_env_limit: { + name: "max_env_limit", + ...column("UInt32", { + description: "The environment concurrency limit. Aggregate with max().", + fillMode: "carry", + }), + }, + max_ck_backlogged: { + name: "max_ck_backlogged", + ...column("UInt32", { + description: + "Peak number of distinct concurrency keys with queued runs in the bucket. Aggregate with max(). Zero for queues that do not use concurrency keys.", + fillMode: "carry", + }), + }, + max_ck_wait_ms: { + name: "max_ck_wait_ms", + ...column("UInt32", { + description: + "Worst head-of-line wait (ms) across concurrency keys in the bucket: how long the most-starved key's oldest queued run has been waiting. Aggregate with max(). Zero for queues that do not use concurrency keys.", + fillMode: "carry", + }), + }, + wait_ms_sum: { + name: "wait_ms_sum", + ...column("UInt64", { + description: "Sum of scheduling delays (ms). Mean = wait_ms_sum/wait_ms_count.", + }), + }, + wait_ms_count: { + name: "wait_ms_count", + ...column("UInt64", { + description: "Count of scheduling-delay samples. Aggregate with sum().", + }), + }, + wait_quantiles: { + name: "wait_quantiles", + ...column("String", { + description: + "Scheduling-delay (dequeue minus eligible-at) quantile state. Read with quantilesMerge(0.5,0.9,0.95,0.99)(wait_quantiles)[n].", + }), + groupable: false, + sortable: false, + filterable: false, + }, + }, + timeBucketThresholds: [ + { maxRangeSeconds: 3 * 60 * 60, interval: { value: 10, unit: "SECOND" } }, + { maxRangeSeconds: 12 * 60 * 60, interval: { value: 1, unit: "MINUTE" } }, + { maxRangeSeconds: 2 * 24 * 60 * 60, interval: { value: 5, unit: "MINUTE" } }, + { maxRangeSeconds: 7 * 24 * 60 * 60, interval: { value: 15, unit: "MINUTE" } }, + { maxRangeSeconds: 30 * 24 * 60 * 60, interval: { value: 1, unit: "HOUR" } }, + { maxRangeSeconds: 90 * 24 * 60 * 60, interval: { value: 6, unit: "HOUR" } }, + { maxRangeSeconds: 180 * 24 * 60 * 60, interval: { value: 1, unit: "DAY" } }, + { maxRangeSeconds: 365 * 24 * 60 * 60, interval: { value: 1, unit: "WEEK" } }, + ] satisfies BucketThreshold[], + // Ranges whose bucket interval is >= 5 minutes read the 5m rollup instead (same + // logical columns, ~30x fewer rows). + rollups: [{ minIntervalSeconds: 300, clickhouseName: "trigger_dev.queue_metrics_5m_v1" }], + queryCache: { ttlSeconds: 30, alignSeconds: 30 }, + queryClient: "queueMetrics", +}; + +/** + * Schema definition for the env_metrics table (trigger_dev.env_metrics_v1). + * Environment-level rollup of queue_metrics with the queue dimension dropped, so + * header tiles and saturation charts cost the same regardless of how many queues + * the environment has. Keeps the full 10-second granularity: row count is + * queue-independent, so even 30-day ranges stay small. + */ +export const envMetricsSchema: TableSchema = { + name: "env_metrics", + clickhouseName: "trigger_dev.env_metrics_v1", + description: + "Environment-level concurrency, saturation, throttling, and scheduling-delay metrics (10-second buckets)", + timeConstraint: "bucket_start", + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + columns: { + environment: { + name: "environment", + clickhouseName: "environment_id", + ...column("String", { description: "The environment slug", example: "prod" }), + fieldMapping: "environment", + customRenderType: "environment", + }, + project: { + name: "project", + clickhouseName: "project_id", + ...column("String", { + description: "The project reference, they always start with `proj_`.", + example: "proj_howcnaxbfxdmwmxazktx", + }), + fieldMapping: "project", + customRenderType: "project", + }, + bucket_start: { + name: "bucket_start", + ...column("DateTime", { + description: "The start of the 10-second aggregation bucket", + example: "2024-01-15 09:30:00", + coreColumn: true, + }), + }, + max_env_queued: { + name: "max_env_queued", + ...column("UInt32", { + description: "Peak environment-wide queued in the bucket. Aggregate with max().", + coreColumn: true, + fillMode: "carry", + }), + }, + max_env_running: { + name: "max_env_running", + ...column("UInt32", { + description: "Peak environment-wide running in the bucket. Aggregate with max().", + coreColumn: true, + fillMode: "carry", + }), + }, + max_env_limit: { + name: "max_env_limit", + ...column("UInt32", { + description: "The environment concurrency limit. Aggregate with max().", + coreColumn: true, + fillMode: "carry", + }), + }, + throttled_count: { + name: "throttled_count", + ...column("UInt64", { + description: + "Gauge emissions where a queue was at its limit with work queued. Aggregate with sum().", + coreColumn: true, + }), + }, + wait_ms_sum: { + name: "wait_ms_sum", + ...column("UInt64", { + description: "Sum of scheduling delays (ms). Mean = wait_ms_sum/wait_ms_count.", + }), + }, + wait_ms_count: { + name: "wait_ms_count", + ...column("UInt64", { + description: "Count of scheduling-delay samples. Aggregate with sum().", + }), + }, + wait_quantiles: { + name: "wait_quantiles", + ...column("String", { + description: + "Scheduling-delay quantile state (TDigest). Read with quantilesTDigestMerge(0.5,0.9,0.95,0.99)(wait_quantiles)[n].", + }), + groupable: false, + sortable: false, + filterable: false, + }, + }, + timeBucketThresholds: [ + { maxRangeSeconds: 3 * 60 * 60, interval: { value: 10, unit: "SECOND" } }, + { maxRangeSeconds: 12 * 60 * 60, interval: { value: 1, unit: "MINUTE" } }, + { maxRangeSeconds: 2 * 24 * 60 * 60, interval: { value: 5, unit: "MINUTE" } }, + { maxRangeSeconds: 7 * 24 * 60 * 60, interval: { value: 15, unit: "MINUTE" } }, + { maxRangeSeconds: 30 * 24 * 60 * 60, interval: { value: 1, unit: "HOUR" } }, + { maxRangeSeconds: 90 * 24 * 60 * 60, interval: { value: 6, unit: "HOUR" } }, + { maxRangeSeconds: 180 * 24 * 60 * 60, interval: { value: 1, unit: "DAY" } }, + { maxRangeSeconds: 365 * 24 * 60 * 60, interval: { value: 1, unit: "WEEK" } }, + ] satisfies BucketThreshold[], + queryCache: { ttlSeconds: 30, alignSeconds: 30 }, + queryClient: "queueMetrics", +}; + /** * Schema definition for the llm_metrics table (trigger_dev.llm_metrics_v1) */ @@ -971,13 +1298,176 @@ export const llmModelsSchema: TableSchema = { }, }; +/** + * Per-concurrency-key drill-down for queues that shard work with `concurrencyKey` + * (e.g. per-tenant fairness). Rows are activity-bound: a (queue, key, bucket) row exists + * only when that key had events, so key cardinality cannot inflate the table. + */ +export const queueMetricsByKeySchema: TableSchema = { + name: "queue_metrics_by_key", + clickhouseName: "trigger_dev.queue_metrics_ck_v1", + description: "Per-concurrency-key queue metrics: backlog, throughput, and wait by key", + hidden: true, + timeConstraint: "bucket_start", + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + columns: { + environment: { + name: "environment", + clickhouseName: "environment_id", + ...column("String", { description: "The environment slug", example: "prod" }), + fieldMapping: "environment", + customRenderType: "environment", + }, + project: { + name: "project", + clickhouseName: "project_id", + ...column("String", { + description: "The project reference, they always start with `proj_`.", + example: "proj_howcnaxbfxdmwmxazktx", + }), + fieldMapping: "project", + customRenderType: "project", + }, + queue: { + name: "queue", + clickhouseName: "queue_name", + ...column("LowCardinality(String)", { + description: "The queue name", + example: "my-queue", + coreColumn: true, + }), + }, + concurrency_key: { + name: "concurrency_key", + ...column("String", { + description: "The concurrency key the run was sharded by (e.g. a tenant id)", + example: "tenant-42", + coreColumn: true, + }), + }, + bucket_start: { + name: "bucket_start", + ...column("DateTime", { + description: "The start of the 10-second aggregation bucket", + example: "2024-01-15 09:30:00", + coreColumn: true, + }), + }, + enqueue_delta: { + name: "enqueue_delta", + mergeGroupKey: ["queue", "concurrency_key"], + ...column("String", { + description: + "Runs enqueued for this key (cumulative-counter delta). Read with deltaSumTimestampMerge(enqueue_delta) grouped by queue and concurrency_key, or with both pinned; never merge across keys.", + }), + groupable: false, + sortable: false, + filterable: false, + }, + started_delta: { + name: "started_delta", + mergeGroupKey: ["queue", "concurrency_key"], + ...column("String", { + description: + "Runs dequeued/started for this key (throughput). Read with deltaSumTimestampMerge(started_delta) grouped by queue and concurrency_key, or with both pinned; never merge across keys.", + coreColumn: true, + }), + groupable: false, + sortable: false, + filterable: false, + }, + ack_delta: { + name: "ack_delta", + mergeGroupKey: ["queue", "concurrency_key"], + ...column("String", { + description: + "Runs acked (completed) for this key. Read with deltaSumTimestampMerge(ack_delta) grouped by queue and concurrency_key, or with both pinned.", + }), + groupable: false, + sortable: false, + filterable: false, + }, + max_queued: { + name: "max_queued", + ...column("UInt32", { + description: "Peak backlog for this key in the bucket. Aggregate with max().", + coreColumn: true, + fillMode: "carry", + }), + }, + max_running: { + name: "max_running", + ...column("UInt32", { + description: "Peak running for this key in the bucket. Aggregate with max().", + fillMode: "carry", + }), + }, + wait_ms_sum: { + name: "wait_ms_sum", + ...column("UInt64", { + description: + "Sum of scheduling delays (ms) for this key. Mean = wait_ms_sum/wait_ms_count.", + }), + }, + wait_ms_count: { + name: "wait_ms_count", + ...column("UInt64", { + description: "Count of scheduling-delay samples for this key. Aggregate with sum().", + }), + }, + }, + timeBucketThresholds: [ + { maxRangeSeconds: 3 * 60 * 60, interval: { value: 10, unit: "SECOND" } }, + { maxRangeSeconds: 12 * 60 * 60, interval: { value: 1, unit: "MINUTE" } }, + { maxRangeSeconds: 2 * 24 * 60 * 60, interval: { value: 5, unit: "MINUTE" } }, + { maxRangeSeconds: 7 * 24 * 60 * 60, interval: { value: 15, unit: "MINUTE" } }, + { maxRangeSeconds: 30 * 24 * 60 * 60, interval: { value: 1, unit: "HOUR" } }, + { maxRangeSeconds: 90 * 24 * 60 * 60, interval: { value: 6, unit: "HOUR" } }, + { maxRangeSeconds: 180 * 24 * 60 * 60, interval: { value: 1, unit: "DAY" } }, + { maxRangeSeconds: 365 * 24 * 60 * 60, interval: { value: 1, unit: "WEEK" } }, + ] satisfies BucketThreshold[], + queryCache: { ttlSeconds: 30, alignSeconds: 30 }, + queryClient: "queueMetrics", +}; + export const querySchemas: TableSchema[] = [ runsSchema, metricsSchema, llmMetricsSchema, llmModelsSchema, + queueMetricsSchema, + envMetricsSchema, + queueMetricsByKeySchema, ]; +/** Tables whose listing is deferred until queue metrics are rolled out. */ +const QUEUE_METRICS_TABLE_NAMES = new Set([ + queueMetricsSchema.name, + envMetricsSchema.name, + queueMetricsByKeySchema.name, +]); + +/** + * Schemas shown in user-facing listings (editor autocomplete, schema docs, schema API, AI query + * context). Listing only: `querySchemas` stays the compile-time set, so a query naming an unlisted + * table still runs, with tenancy enforced as usual. + * + * Server callers pass `env.QUEUE_METRICS_QUERY_TABLES_VISIBLE === "1"`; client callers read + * `queueMetricsQueryTables` from `useFeatures()`. This module is imported by browser code, so it + * must not reach for `env.server` itself. + */ +export function listableQuerySchemas(options: { includeQueueMetrics: boolean }): TableSchema[] { + return querySchemas.filter((s) => { + if (s.hidden) return false; + if (!options.includeQueueMetrics && QUEUE_METRICS_TABLE_NAMES.has(s.name)) return false; + return true; + }); +} + /** * Default query for the query editor */ diff --git a/apps/webapp/app/v3/queueMetrics.server.ts b/apps/webapp/app/v3/queueMetrics.server.ts new file mode 100644 index 00000000000..2df3b255b60 --- /dev/null +++ b/apps/webapp/app/v3/queueMetrics.server.ts @@ -0,0 +1,247 @@ +import { type ClickHouse, type QueueMetricsRawV1Input } from "@internal/clickhouse"; +import { + allStreamKeys, + CachedRedisFlag, + CachedRedisNumber, + MetricsStreamConsumer, + MetricsStreamEmitter, + probeShardStates, + type MetricDefinition, + type ShardState, + type StreamEntry, +} from "@internal/metrics-pipeline"; +import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis"; +import os from "node:os"; +import { env } from "~/env.server"; +import { getQueueMetricsClickhouseClient } from "~/services/clickhouse/clickhouseFactory.server"; +import { logger } from "~/services/logger.server"; +import { signalsEmitter } from "~/services/signals.server"; +import { singleton } from "~/utils/singleton"; +import { mapEntryToRows, QueueNameLimiter } from "./queueMetricsMapping"; +import { meter } from "./tracer.server"; + +const FLAG_KEY = "queue_metrics:enabled"; +const SAMPLE_RATE_KEY = "queue_metrics:gauge_sample_rate"; +const TRUTHY = new Set(["1", "true", "on", "enabled", "yes"]); + +// Same physical Redis as the RunQueue (host/port/auth). Stream keys are kept out of the +// keyPrefix on every access path, so only the connection details matter here. +function runQueueRedisOptions(): RedisOptions { + return { + port: env.RUN_ENGINE_RUN_QUEUE_REDIS_PORT ?? undefined, + host: env.RUN_ENGINE_RUN_QUEUE_REDIS_HOST ?? undefined, + username: env.RUN_ENGINE_RUN_QUEUE_REDIS_USERNAME ?? undefined, + password: env.RUN_ENGINE_RUN_QUEUE_REDIS_PASSWORD ?? undefined, + enableAutoPipelining: true, + ...(env.RUN_ENGINE_RUN_QUEUE_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }), + }; +} + +// Metrics stream Redis: a dedicated instance when QUEUE_METRICS_REDIS_HOST is set (so the +// metrics backlog never competes with the run queue), else the run-queue Redis. Carries BOTH +// gauges and counters — gauges are read inside the queue-op Lua and returned on the reply, +// then XADDed here by Node, so the run-queue Redis holds no metrics stream. +function metricsRedisOptions(): RedisOptions { + if (!env.QUEUE_METRICS_REDIS_HOST) return runQueueRedisOptions(); + return { + host: env.QUEUE_METRICS_REDIS_HOST, + port: env.QUEUE_METRICS_REDIS_PORT ?? undefined, + username: env.QUEUE_METRICS_REDIS_USERNAME ?? undefined, + password: env.QUEUE_METRICS_REDIS_PASSWORD ?? undefined, + enableAutoPipelining: true, + ...(env.QUEUE_METRICS_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }), + }; +} + +// One stream family on the metrics Redis carrying both gauge snapshots and cumulative +// counter readings; one consumer group reads it. +function metricsDefinition(): MetricDefinition { + // A stalled consumer holds up to maxLen entries per shard in Redis memory: cap lower + // by default when the stream shares the queue-critical run-queue Redis. + const defaultMaxLen = env.QUEUE_METRICS_REDIS_HOST ? 8_000_000 : 2_000_000; + return { + name: "queue_metrics", + shardCount: env.QUEUE_METRICS_STREAM_SHARD_COUNT, + consumerGroup: "queue_metrics_cg", + maxLen: env.QUEUE_METRICS_COUNTER_STREAM_MAXLEN ?? defaultMaxLen, + }; +} + +// Dedicated client for the admin read/write/probe surface — works regardless of whether +// this instance runs the emitter/consumer. keyPrefix unset to match the raw control keys. +function adminRedis(): Redis { + return singleton("queueMetricsAdminRedis", () => + createRedisClient( + { ...runQueueRedisOptions(), keyPrefix: undefined }, + { onError: (error) => logger.error("queue metrics admin redis error", { error }) } + ) + ); +} + +function metricsAdminRedis(): Redis { + return singleton("queueMetricsCounterAdminRedis", () => + createRedisClient( + { ...metricsRedisOptions(), keyPrefix: undefined }, + { onError: (error) => logger.error("queue metrics counter admin redis error", { error }) } + ) + ); +} + +export type QueueMetricsControls = { + enabled: boolean; + enabledKeySet: boolean; + sampleRate: number; + sampleRateKeySet: boolean; + sampleRateDefault: number; +}; + +export async function readQueueMetricsControls(): Promise<QueueMetricsControls> { + const [enabledRaw, rateRaw] = (await adminRedis().mget(FLAG_KEY, SAMPLE_RATE_KEY)) as ( + | string + | null + )[]; + const sampleRateDefault = env.QUEUE_METRICS_GAUGE_SAMPLE_RATE; + const parsed = rateRaw == null ? Number.NaN : Number(rateRaw); + return { + enabled: enabledRaw != null && TRUTHY.has(enabledRaw.trim().toLowerCase()), + enabledKeySet: enabledRaw != null, + sampleRate: Number.isFinite(parsed) ? Math.min(1, Math.max(0, parsed)) : sampleRateDefault, + sampleRateKeySet: rateRaw != null, + sampleRateDefault, + }; +} + +export async function writeQueueMetricsControls(update: { + enabled?: boolean; + sampleRate?: number; +}): Promise<void> { + const client = adminRedis(); + const ops: Promise<unknown>[] = []; + if (update.enabled !== undefined) { + ops.push(client.set(FLAG_KEY, update.enabled ? "1" : "0")); + } + if (update.sampleRate !== undefined) { + ops.push(client.set(SAMPLE_RATE_KEY, String(Math.min(1, Math.max(0, update.sampleRate))))); + } + await Promise.all(ops); +} + +export type LabeledShardState = ShardState & { stream: "queue_metrics" }; + +export async function probeQueueMetricsStreams(): Promise<LabeledShardState[]> { + const def = metricsDefinition(); + const states = await probeShardStates(metricsAdminRedis(), allStreamKeys(def), def.consumerGroup); + return states.map((s) => ({ ...s, stream: "queue_metrics" as const })); +} + +/** Injected into the RunQueue when QUEUE_METRICS_EMIT_ENABLED=1; emits only while the flag is on. */ +export function getQueueMetricsEmitter(): MetricsStreamEmitter { + return singleton("queueMetricsEmitter", () => { + // Control keys stay on the run-queue Redis (the admin surface + docs point there). + const controlRedis = runQueueRedisOptions(); + const flag = new CachedRedisFlag({ redis: controlRedis, key: FLAG_KEY, cacheTtlMs: 10_000 }); + // Live-tunable (Redis key, 10s cache); the env value is the default when the key is unset. + const gaugeSampleRate = new CachedRedisNumber({ + redis: controlRedis, + key: SAMPLE_RATE_KEY, + defaultValue: env.QUEUE_METRICS_GAUGE_SAMPLE_RATE, + min: 0, + max: 1, + cacheTtlMs: 10_000, + }); + return new MetricsStreamEmitter({ + redis: metricsRedisOptions(), + definition: metricsDefinition(), + flag, + meter, + gaugeSampleRate, + counterOdometerTtlMs: env.QUEUE_METRICS_COUNTER_ODOMETER_TTL_SECONDS * 1000, + }); + }); +} + +const queueNameLimiter = singleton( + "queueMetricsQueueNameLimiter", + () => new QueueNameLimiter(env.QUEUE_METRICS_MAX_QUEUE_NAMES_PER_ENV) +); + +const concurrencyKeyLimiter = singleton( + "queueMetricsConcurrencyKeyLimiter", + () => new QueueNameLimiter(env.QUEUE_METRICS_MAX_CONCURRENCY_KEYS_PER_QUEUE, 50_000) +); + +function mapEntry(entry: StreamEntry): QueueMetricsRawV1Input[] { + return mapEntryToRows(entry, { + queueNames: queueNameLimiter, + concurrencyKeys: concurrencyKeyLimiter, + }); +} + +function makeInsert(): ( + rows: QueueMetricsRawV1Input[], + opts: { dedupToken: string } +) => Promise<void> { + const ch: ClickHouse = getQueueMetricsClickhouseClient(); + const insertRaw = ch.queueMetrics.insertRaw; + return async (rows, { dedupToken }) => { + const [error] = await insertRaw(rows, { + params: { + clickhouse_settings: { + insert_deduplication_token: dedupToken, + async_insert: 0, + // Propagate the token through the MV so a raw-deduped retry can't leave + // queue_metrics_v1 short when the MV insert failed on the first attempt. + deduplicate_blocks_in_dependent_materialized_views: 1, + }, + }, + }); + if (error) throw error; + }; +} + +function getQueueMetricsConsumers(): MetricsStreamConsumer<QueueMetricsRawV1Input>[] { + return singleton("queueMetricsConsumers", () => { + const insert = makeInsert(); + return [ + new MetricsStreamConsumer<QueueMetricsRawV1Input>({ + consumerName: `${os.hostname()}-${process.pid}`, + batchSize: env.QUEUE_METRICS_CONSUMER_BATCH_SIZE, + meter, + mapEntry, + insert, + redis: metricsRedisOptions(), + definition: metricsDefinition(), + }), + ]; + }); +} + +// Construct the emitter at boot (not lazily on the first enqueue) so its flag has warmed +// before any traffic — otherwise the first op after boot reads the default and is dropped. +export function initQueueMetricsEmitter(): void { + if (env.QUEUE_METRICS_EMIT_ENABLED !== "1") return; + getQueueMetricsEmitter(); +} + +declare global { + // eslint-disable-next-line no-var + var __queueMetricsConsumerRegistered__: boolean | undefined; +} + +export function initQueueMetricsConsumer(): void { + if (env.QUEUE_METRICS_CONSUMER_ENABLED !== "1") return; + if (global.__queueMetricsConsumerRegistered__) return; + global.__queueMetricsConsumerRegistered__ = true; + + const consumers = getQueueMetricsConsumers(); + const stop = () => + Promise.all(consumers.map((c) => c.stop())).catch((error) => + logger.error("queue metrics consumer stop failed", { error }) + ); + signalsEmitter.on("SIGTERM", stop); + signalsEmitter.on("SIGINT", stop); + + Promise.all(consumers.map((c) => c.start())) + .then(() => logger.info("Queue metrics consumer started")) + .catch((error) => logger.error("queue metrics consumers failed to start", { error })); +} diff --git a/apps/webapp/app/v3/queueMetricsMapping.ts b/apps/webapp/app/v3/queueMetricsMapping.ts new file mode 100644 index 00000000000..9433b361a88 --- /dev/null +++ b/apps/webapp/app/v3/queueMetricsMapping.ts @@ -0,0 +1,164 @@ +import { type QueueMetricsRawV1Input } from "@internal/clickhouse"; +import { entryOrderKey, entryTimeMs, type StreamEntry } from "@internal/metrics-pipeline"; + +const OPS = new Set(["gauge", "enqueue", "started", "ack", "nack", "dlq"]); + +// {org:ORGID}:proj:PROJECTID:env:ENVID:queue:QUEUENAME[:ck:CK]. Anchored (not a +// positional split) so a queue name containing ":" survives; the lazy name capture +// stops before an optional ":ck:" suffix, which is captured (the ":ck:*" wildcard of +// aggregate CK-dequeue gauges maps to no key). +const DESCRIPTOR = /^\{org:([^}]+)\}:proj:([^:]+):env:([^:]+):queue:(.+?)(?::ck:(.+))?$/; + +export function descriptorFromQueue(q: string): { + organization_id: string; + project_id: string; + environment_id: string; + queue_name: string; + concurrency_key: string; +} | null { + const match = DESCRIPTOR.exec(q); + if (!match) return null; + const ck = match[5]; + return { + organization_id: match[1]!, + project_id: match[2]!, + environment_id: match[3]!, + queue_name: match[4]!, + concurrency_key: ck && ck !== "*" ? ck : "", + }; +} + +export const OVERFLOW_QUEUE_NAME = "__overflow__"; + +/** + * Bounds per-scope name cardinality (both queue_name per env and concurrency_key per + * queue are user-controlled GROUP BY keys). Names beyond the cap map to OVERFLOW_QUEUE_NAME. + * Per-process and reset on restart, so the cap is approximate: a protective bound, not a quota. + */ +export class QueueNameLimiter { + private readonly byScope = new Map<string, Set<string>>(); + + constructor( + private readonly maxPerScope: number, + private readonly maxScopes = 10_000 + ) {} + + limit(scope: string, name: string): string { + if (this.maxPerScope <= 0) return name; + let names = this.byScope.get(scope); + if (!names) { + if (this.byScope.size >= this.maxScopes) { + const oldest = this.byScope.keys().next().value; + if (oldest !== undefined) this.byScope.delete(oldest); + } + names = new Set(); + this.byScope.set(scope, names); + } + if (names.has(name)) return name; + if (names.size >= this.maxPerScope) return OVERFLOW_QUEUE_NAME; + names.add(name); + return name; + } +} + +function num(value: string | undefined): number | undefined { + if (value == null) return undefined; + const n = Number(value); + return Number.isFinite(n) ? n : undefined; +} + +export type QueueMetricsLimiters = { + queueNames?: QueueNameLimiter; + concurrencyKeys?: QueueNameLimiter; +}; + +/** + * One stream entry maps to 1..2 raw rows: gauges are single rows carrying their parsed + * concurrency_key; a counter entry yields a base row when `cum` is present plus a per-key + * row when `ck`/`ckcum` are present (the emitter's dual-odometer entry). Baseline entries + * carry only one of the two, by design. + */ +export function mapEntryToRows( + entry: StreamEntry, + limiters?: QueueMetricsLimiters +): QueueMetricsRawV1Input[] { + const f = entry.fields; + const op = f.op; + if (!op || !OPS.has(op) || !f.q) return []; + const descriptor = descriptorFromQueue(f.q); + if (!descriptor || !descriptor.queue_name) return []; + + let queueOverflowed = false; + if (limiters?.queueNames) { + descriptor.queue_name = limiters.queueNames.limit( + descriptor.environment_id, + descriptor.queue_name + ); + queueOverflowed = descriptor.queue_name === OVERFLOW_QUEUE_NAME; + } + + // Counter entries carry the key as a field (q is base-normalized); gauges carry it in q. + let ck = descriptor.concurrency_key || (typeof f.ck === "string" ? f.ck : ""); + if (ck && limiters?.concurrencyKeys) { + const scope = `${descriptor.environment_id}:${descriptor.queue_name}`; + if (limiters.concurrencyKeys.limit(scope, ck) === OVERFLOW_QUEUE_NAME) ck = ""; + } + // Overflowed queue names share one row; per-key attribution under them is meaningless. + if (queueOverflowed) ck = ""; + + const eventMs = entryTimeMs(entry.id) ?? Date.now(); + const eventTime = new Date(eventMs).toISOString().slice(0, 19).replace("T", " "); + const base = { + organization_id: descriptor.organization_id, + project_id: descriptor.project_id, + environment_id: descriptor.environment_id, + queue_name: descriptor.queue_name, + event_time: eventTime, + op: op as QueueMetricsRawV1Input["op"], + }; + + if (op === "gauge") { + return [ + { + ...base, + concurrency_key: ck, + queued: num(f.ql), + running: num(f.cc), + queue_limit: num(f.lim), + env_queued: num(f.eql), + env_running: num(f.ec), + env_limit: num(f.elim), + throttled: num(f.thr), + ck_backlogged: num(f.ckq), + ck_max_wait_ms: num(f.ckw), + }, + ]; + } + + // Overflowed names drop counters entirely: merging distinct odometers under one shared + // name produces garbage deltas (gauges above stay, max across the overflow set is + // still meaningful). + if (queueOverflowed) return []; + + const rows: QueueMetricsRawV1Input[] = []; + const orderKey = entryOrderKey(entry.id); + const waitMs = op === "started" && f.wait != null ? num(f.wait) : undefined; + if (f.cum != null) { + rows.push({ + ...base, + cumulative: num(f.cum), + order_key: orderKey, + ...(waitMs !== undefined ? { wait_ms: waitMs } : {}), + }); + } + if (ck && f.ckcum != null) { + rows.push({ + ...base, + concurrency_key: ck, + cumulative: num(f.ckcum), + order_key: orderKey, + ...(waitMs !== undefined ? { wait_ms: waitMs } : {}), + }); + } + return rows; +} diff --git a/apps/webapp/app/v3/queueSparklineGrid.ts b/apps/webapp/app/v3/queueSparklineGrid.ts new file mode 100644 index 00000000000..7fce15bc962 --- /dev/null +++ b/apps/webapp/app/v3/queueSparklineGrid.ts @@ -0,0 +1,54 @@ +/** + * Bucket grid for the Queues list sparklines. Pure so it can be unit tested: the presenter that + * uses it reaches ClickHouse, and the grid arithmetic is where the off-by-one lives. + * + * The grid start floors to a bucket boundary and the end ceils to one, so repeated loads inside a + * bucket produce identical query params and share ClickHouse query-cache entries. That means the + * span covered by the grid is wider than the requested range whenever `from` is not already on a + * boundary, which is almost always. `bucketCount` is therefore measured from the aligned grid + * start rather than from the raw range, otherwise the newest bucket lands at an index the caller + * treats as out of range and its data is silently dropped. + */ + +export const SPARKLINE_POINTS = 48; + +const MIN_RANGE_SECONDS = 60; +const MIN_BUCKET_SECONDS = 60; + +export type SparklineGrid = { + bucketSeconds: number; + bucketIntervalMs: number; + /** Aligned grid start; at or before the requested `from`. */ + bucketStartMs: number; + /** Aligned grid end; at or after the requested `to`. */ + endMs: number; + /** Buckets spanning bucketStartMs..endMs, so every bucket the query can return has an index. */ + bucketCount: number; +}; + +export function computeSparklineGrid(from: Date, to: Date): SparklineGrid { + const rangeSeconds = Math.max( + MIN_RANGE_SECONDS, + Math.round((to.getTime() - from.getTime()) / 1000) + ); + const bucketSeconds = Math.max(MIN_BUCKET_SECONDS, Math.round(rangeSeconds / SPARKLINE_POINTS)); + const bucketIntervalMs = bucketSeconds * 1000; + + const gridStartSeconds = + Math.floor(Math.floor(from.getTime() / 1000) / bucketSeconds) * bucketSeconds; + const bucketStartMs = gridStartSeconds * 1000; + const endMs = Math.ceil(to.getTime() / bucketIntervalMs) * bucketIntervalMs; + + const bucketCount = Math.max(1, Math.round((endMs - bucketStartMs) / bucketIntervalMs)); + + return { bucketSeconds, bucketIntervalMs, bucketStartMs, endMs, bucketCount }; +} + +/** Index of a bucket on the grid, or null when it falls outside it. */ +export function bucketIndex(grid: SparklineGrid, bucketMs: number): number | null { + const index = Math.round((bucketMs - grid.bucketStartMs) / grid.bucketIntervalMs); + if (index < 0 || index >= grid.bucketCount) { + return null; + } + return index; +} diff --git a/apps/webapp/app/v3/runEngine.server.ts b/apps/webapp/app/v3/runEngine.server.ts index 4d9e263d6be..85986933290 100644 --- a/apps/webapp/app/v3/runEngine.server.ts +++ b/apps/webapp/app/v3/runEngine.server.ts @@ -7,6 +7,7 @@ import { logger } from "~/services/logger.server"; import { defaultMachine, getCurrentPlan } from "~/services/platform.v3.server"; import { singleton } from "~/utils/singleton"; import { allMachines } from "./machinePresets.server"; +import { getQueueMetricsEmitter } from "./queueMetrics.server"; import { runEnginePendingVersionLookup } from "./runEnginePendingVersionLookup.server"; import { pickRunOpsStoreForCompletion } from "./runOpsMigration/crossSeamGuard.server"; import { runEngineControlPlaneResolver } from "./runOpsMigration/runEngineControlPlaneResolver.server"; @@ -83,6 +84,7 @@ function createRunEngine() { tracer, }, shardCount: env.RUN_ENGINE_RUN_QUEUE_SHARD_COUNT, + queueMetrics: env.QUEUE_METRICS_EMIT_ENABLED === "1" ? getQueueMetricsEmitter() : undefined, processWorkerQueueDebounceMs: env.RUN_ENGINE_PROCESS_WORKER_QUEUE_DEBOUNCE_MS, dequeueBlockingTimeoutSeconds: env.RUN_ENGINE_DEQUEUE_BLOCKING_TIMEOUT_SECONDS, masterQueueConsumersIntervalMs: env.RUN_ENGINE_MASTER_QUEUE_CONSUMERS_INTERVAL_MS, diff --git a/apps/webapp/app/v3/services/allocateConcurrency.server.ts b/apps/webapp/app/v3/services/allocateConcurrency.server.ts index b9a28daba60..78c8b82915a 100644 --- a/apps/webapp/app/v3/services/allocateConcurrency.server.ts +++ b/apps/webapp/app/v3/services/allocateConcurrency.server.ts @@ -3,6 +3,7 @@ import { ManageConcurrencyPresenter } from "~/presenters/v3/ManageConcurrencyPre import { BaseService } from "./baseService.server"; import { updateEnvConcurrencyLimits } from "../runQueue.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; +import { concurrencySystem } from "./concurrencySystemInstance.server"; type Input = { userId: string; @@ -90,6 +91,14 @@ export class AllocateConcurrencyService extends BaseService { await updateEnvConcurrencyLimits(updatedEnvironment); } + // Percent-based queue overrides follow the environment limit automatically. Note the + // deliberate asymmetry with the env-level push above: `updateEnvConcurrencyLimits` is gated + // on `!paused`, but we recalculate queue limits even for paused environments. Queue-level + // pushes on a paused env are inert (the env-level gate stops dequeueing regardless), and + // keeping the queue limits synced means resume needs no extra reconciliation — skipping + // them here would instead leave stale engine limits after the env resumes. + await concurrencySystem.queues.recalculatePercentLimits(updatedEnvironment); + // maximumConcurrencyLimit changed in the control-plane; drop any cached copy. controlPlaneResolver.invalidateEnvironment(environment.id); } diff --git a/apps/webapp/app/v3/services/concurrencySystem.server.ts b/apps/webapp/app/v3/services/concurrencySystem.server.ts index 488f38ce279..f030cb72e5f 100644 --- a/apps/webapp/app/v3/services/concurrencySystem.server.ts +++ b/apps/webapp/app/v3/services/concurrencySystem.server.ts @@ -2,6 +2,7 @@ import type { TaskQueue, User } from "@trigger.dev/database"; import { errAsync, fromPromise, okAsync } from "neverthrow"; import type { PrismaClientOrTransaction } from "~/db.server"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { logger } from "~/services/logger.server"; import { removeQueueConcurrencyLimits, updateQueueConcurrencyLimits } from "../runQueue.server"; import { engine } from "../runEngine.server"; @@ -12,6 +13,42 @@ export type ConcurrencySystemOptions = { export type QueueInput = string | { type: "task" | "custom"; name: string }; +/** + * The concurrency-limit override to apply to a queue. Either an absolute `limit` or a `percent` + * of the environment's maximum concurrency limit. A bare `number` is accepted for backwards + * compatibility and is treated as an absolute limit. + */ +export type ConcurrencyLimitOverride = number | { limit: number } | { percent: number }; + +/** + * Materializes an absolute concurrency limit from a percentage of the environment limit. + * Reused by the recalculation task that runs when an environment limit changes. + * + * Clamped to `>= 1` so a percent-based override never produces a `0` (pause-like) limit, and + * to `<= envLimit` so it can never exceed the environment maximum. + */ +/** + * Valid range for a percent-based queue concurrency override: greater than 0 and up to 100% of + * the environment limit. Shared by every layer that validates the percent (the API zod schema, + * the dashboard mutation handler, and the override service) so the bound never drifts apart. + */ +export const MIN_QUEUE_OVERRIDE_PERCENT = 0; +export const MAX_QUEUE_OVERRIDE_PERCENT = 100; + +/** Whether `percent` is a valid queue-override percentage (0 < percent <= 100). */ +export function isValidQueueOverridePercent(percent: number): boolean { + return ( + Number.isFinite(percent) && + percent > MIN_QUEUE_OVERRIDE_PERCENT && + percent <= MAX_QUEUE_OVERRIDE_PERCENT + ); +} + +export function materializePercentLimit(envLimit: number, percent: number): number { + const materialized = Math.floor((envLimit * percent) / 100); + return Math.min(Math.max(materialized, 1), envLimit); +} + export class ConcurrencySystem { constructor(private readonly options: ConcurrencySystemOptions) {} @@ -24,18 +61,12 @@ export class ConcurrencySystem { overrideQueueConcurrencyLimit: ( environment: AuthenticatedEnvironment, queue: QueueInput, - concurrencyLimit: number, + override: ConcurrencyLimitOverride, overriddenBy?: User ) => { return findQueueFromInput(this.db, environment, queue) .andThen((queue) => - overrideQueueConcurrencyLimit( - this.db, - environment, - queue, - concurrencyLimit, - overriddenBy - ) + overrideQueueConcurrencyLimit(this.db, environment, queue, override, overriddenBy) ) .andThen((queue) => syncQueueConcurrencyToEngine(environment, queue)) .andThen((queue) => getQueueStats(environment, queue)); @@ -46,6 +77,59 @@ export class ConcurrencySystem { .andThen((queue) => syncQueueConcurrencyToEngine(environment, queue)) .andThen((queue) => getQueueStats(environment, queue)); }, + /** + * Recalculates the materialized limit of every percent-based override in the environment + * against its CURRENT maximumConcurrencyLimit and syncs changed queues to the run engine. + * Call AFTER the environment-limit DB update has committed (engine syncs must not run + * inside an open transaction). Idempotent: unchanged queues are skipped. One failing queue + * is logged and skipped so the rest still converge. + */ + recalculatePercentLimits: async (environment: AuthenticatedEnvironment) => { + const queues = await this.db.taskQueue.findMany({ + where: { + runtimeEnvironmentId: environment.id, + concurrencyLimitOverridePercent: { not: null }, + }, + }); + + let updated = 0; + for (const queue of queues) { + try { + const percent = queue.concurrencyLimitOverridePercent; + if (percent === null) continue; + const newLimit = materializePercentLimit( + environment.maximumConcurrencyLimit, + percent.toNumber() + ); + + // Only write the DB when the materialized value actually changed. + if (newLimit !== queue.concurrencyLimit) { + await this.db.taskQueue.update({ + where: { id: queue.id }, + data: { concurrencyLimit: newLimit }, + }); + updated++; + } + + // Always attempt the engine push (it's idempotent) for active queues — even when the + // DB value was unchanged — so a previously-failed sync self-heals on the next recalc + // instead of leaving the DB and engine diverged forever. Paused queues keep their + // engine limit at 0 (the pause/resume flow re-syncs from the stored value on resume); + // push nothing for them so a percent recalc never effectively un-pauses a queue. + if (!queue.paused) { + await updateQueueConcurrencyLimits(environment, queue.name, newLimit); + } + } catch (error) { + logger.error("Failed to recalculate percent queue limit", { + queueId: queue.id, + environmentId: environment.id, + error, + }); + } + } + + return { total: queues.length, updated }; + }, }; } } @@ -117,13 +201,49 @@ function overrideQueueConcurrencyLimit( db: PrismaClientOrTransaction, environment: AuthenticatedEnvironment, queue: TaskQueue, - concurrencyLimit: number, + override: ConcurrencyLimitOverride, overriddenBy?: User ) { - const newConcurrencyLimit = Math.max( - Math.min(concurrencyLimit, environment.maximumConcurrencyLimit), - 0 - ); + const maximum = environment.maximumConcurrencyLimit; + + // Normalize the input into the absolute limit to persist and the percent source-of-truth + // (null for absolute overrides). + let newConcurrencyLimit: number; + let overridePercent: number | null; + + if (typeof override === "object" && "percent" in override) { + const percent = override.percent; + + if (!isValidQueueOverridePercent(percent)) { + return errAsync({ + type: "invalid_override" as const, + message: `Percent must be greater than ${MIN_QUEUE_OVERRIDE_PERCENT} and less than or equal to ${MAX_QUEUE_OVERRIDE_PERCENT}`, + }); + } + + newConcurrencyLimit = materializePercentLimit(maximum, percent); + overridePercent = percent; + } else { + const limit = typeof override === "number" ? override : override.limit; + + if (!Number.isFinite(limit) || limit < 0) { + return errAsync({ + type: "invalid_override" as const, + message: "Concurrency limit must be a non-negative number", + }); + } + + // Cap: an absolute override may not exceed the environment limit. Reject rather than clamp. + if (limit > maximum) { + return errAsync({ + type: "concurrency_limit_exceeds_maximum" as const, + message: `Concurrency limit (${limit}) cannot exceed the environment limit (${maximum})`, + }); + } + + newConcurrencyLimit = limit; + overridePercent = null; + } const concurrencyLimitBase = queue.concurrencyLimitOverriddenAt ? queue.concurrencyLimitBase @@ -137,6 +257,7 @@ function overrideQueueConcurrencyLimit( data: { concurrencyLimit: newConcurrencyLimit, concurrencyLimitBase: concurrencyLimitBase ?? null, + concurrencyLimitOverridePercent: overridePercent, concurrencyLimitOverriddenAt: new Date(), concurrencyLimitOverriddenBy: overriddenBy?.id ?? null, }, @@ -162,6 +283,7 @@ function resetQueueConcurrencyLimit(db: PrismaClientOrTransaction, queue: TaskQu concurrencyLimitOverriddenAt: null, concurrencyLimit: newConcurrencyLimit, concurrencyLimitBase: null, + concurrencyLimitOverridePercent: null, concurrencyLimitOverriddenBy: null, }, }), diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 8819f322f93..c239f5339f8 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -55,6 +55,7 @@ "@internal/dashboard-agent": "workspace:*", "@internal/dashboard-agent-db": "workspace:*", "@internal/llm-model-catalog": "workspace:*", + "@internal/metrics-pipeline": "workspace:*", "@internal/redis": "workspace:*", "@internal/run-engine": "workspace:*", "@internal/run-ops-database": "workspace:*", diff --git a/apps/webapp/test/concurrencySystemPercentOverride.test.ts b/apps/webapp/test/concurrencySystemPercentOverride.test.ts new file mode 100644 index 00000000000..f81f1b1d10b --- /dev/null +++ b/apps/webapp/test/concurrencySystemPercentOverride.test.ts @@ -0,0 +1,370 @@ +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, it, vi } from "vitest"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { ConcurrencySystem, materializePercentLimit } from "~/v3/services/concurrencySystem.server"; + +// The real run engine opens eager Redis connections and needs the whole engine +// wired up. These tests exercise the DB-write + recalculation logic against a +// real Postgres (postgresTest), so we replace the engine singleton with a thin +// stub that satisfies the sync + stats calls the service makes. The engine sync +// itself (RunQueue/Redis) is therefore NOT exercised here — see the note in the +// verification report. Everything the service persists to Postgres IS real. +// A controllable spy for the engine's queue-limit push so a test can simulate a push failure and +// prove the next recalc self-heals the divergence. +// This mocks the ENGINE method (`engine.runQueue.updateQueueConcurrencyLimits`). The service calls +// the top-level `updateQueueConcurrencyLimits` from `~/v3/runQueue.server`, which forwards straight +// to this engine method with the same `(environment, queueName, concurrency)` argument order (no +// transformation) — so mocking the engine method genuinely exercises the recalc's sync path and the +// `(env, queueName, 10)` assertion below is exact. +const { updateQueueConcurrencyLimitsMock } = vi.hoisted(() => ({ + updateQueueConcurrencyLimitsMock: vi.fn(async (..._args: unknown[]) => undefined), +})); + +vi.mock("~/v3/runEngine.server", () => ({ + engine: { + lengthOfQueues: async () => ({}), + currentConcurrencyOfQueues: async () => ({}), + runQueue: { + updateQueueConcurrencyLimits: updateQueueConcurrencyLimitsMock, + removeQueueConcurrencyLimits: async () => undefined, + updateEnvConcurrencyLimits: async () => undefined, + }, + }, +})); + +vi.setConfig({ testTimeout: 30_000 }); + +describe("materializePercentLimit", () => { + it("floors the materialized value", () => { + // 10 * 55 / 100 = 5.5 -> floor -> 5 + expect(materializePercentLimit(10, 55)).toBe(5); + // 7 * 50 / 100 = 3.5 -> floor -> 3 + expect(materializePercentLimit(7, 50)).toBe(3); + }); + + it("clamps to at least 1 so a percent override never produces a 0 (pause-like) limit", () => { + // 10 * 1 / 100 = 0.1 -> floor -> 0 -> clamp up to 1 + expect(materializePercentLimit(10, 1)).toBe(1); + // tiny env, small percent still floors to 0 then clamps to 1 + expect(materializePercentLimit(1, 1)).toBe(1); + expect(materializePercentLimit(3, 10)).toBe(1); + }); + + it("clamps to at most the environment limit at 100%", () => { + expect(materializePercentLimit(10, 100)).toBe(10); + expect(materializePercentLimit(1, 100)).toBe(1); + expect(materializePercentLimit(250, 100)).toBe(250); + }); + + it("handles a tiny environment limit at the 100% boundary", () => { + expect(materializePercentLimit(1, 50)).toBe(1); // 0.5 -> 0 -> clamp to 1 + expect(materializePercentLimit(2, 50)).toBe(1); // 1.0 -> 1 + expect(materializePercentLimit(2, 100)).toBe(2); + }); + + it("supports fractional percentages", () => { + // 100 * 12.5 / 100 = 12.5 -> floor -> 12 + expect(materializePercentLimit(100, 12.5)).toBe(12); + }); +}); + +async function seedEnvAndQueue( + prisma: PrismaClient, + opts: { maximumConcurrencyLimit: number; queueConcurrencyLimit?: number | null } +) { + const slug = `s${Math.random().toString(36).slice(2, 10)}`; + + const organization = await prisma.organization.create({ + data: { title: slug, slug }, + }); + + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: slug }, + }); + + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug, + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: slug, + pkApiKey: slug, + shortcode: slug, + maximumConcurrencyLimit: opts.maximumConcurrencyLimit, + }, + }); + + const queue = await prisma.taskQueue.create({ + data: { + friendlyId: `queue_${slug}`, + name: `task/${slug}`, + projectId: project.id, + runtimeEnvironmentId: environment.id, + concurrencyLimit: opts.queueConcurrencyLimit ?? null, + }, + }); + + const authEnv = { + id: environment.id, + maximumConcurrencyLimit: environment.maximumConcurrencyLimit, + } as unknown as AuthenticatedEnvironment; + + return { organization, project, environment, queue, authEnv }; +} + +describe("ConcurrencySystem percent overrides", () => { + postgresTest( + "materializes a percent override and stores the percent as the source of truth", + async ({ prisma }) => { + const { queue, authEnv } = await seedEnvAndQueue(prisma, { + maximumConcurrencyLimit: 10, + queueConcurrencyLimit: 8, + }); + + const system = new ConcurrencySystem({ db: prisma, reader: prisma }); + + const result = await system.queues.overrideQueueConcurrencyLimit(authEnv, queue.friendlyId, { + percent: 50, + }); + + expect(result.isOk()).toBe(true); + + const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } }); + // floor(10 * 50 / 100) = 5 + expect(row.concurrencyLimit).toBe(5); + expect(row.concurrencyLimitOverridePercent?.toNumber()).toBe(50); + // base captures the pre-override absolute limit + expect(row.concurrencyLimitBase).toBe(8); + expect(row.concurrencyLimitOverriddenAt).not.toBeNull(); + } + ); + + postgresTest( + "rejects an absolute override that exceeds the environment limit and persists nothing", + async ({ prisma }) => { + const { queue, authEnv } = await seedEnvAndQueue(prisma, { + maximumConcurrencyLimit: 10, + queueConcurrencyLimit: 8, + }); + + const system = new ConcurrencySystem({ db: prisma, reader: prisma }); + + const result = await system.queues.overrideQueueConcurrencyLimit(authEnv, queue.friendlyId, { + limit: 9999, + }); + + expect(result.isErr()).toBe(true); + if (result.isErr()) { + expect(result.error.type).toBe("concurrency_limit_exceeds_maximum"); + } + + // Nothing should have been written. + const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } }); + expect(row.concurrencyLimit).toBe(8); + expect(row.concurrencyLimitOverridePercent).toBeNull(); + expect(row.concurrencyLimitOverriddenAt).toBeNull(); + expect(row.concurrencyLimitBase).toBeNull(); + } + ); + + postgresTest("rejects an out-of-range percent as invalid_override", async ({ prisma }) => { + const { queue, authEnv } = await seedEnvAndQueue(prisma, { + maximumConcurrencyLimit: 10, + }); + + const system = new ConcurrencySystem({ db: prisma, reader: prisma }); + + const tooHigh = await system.queues.overrideQueueConcurrencyLimit(authEnv, queue.friendlyId, { + percent: 101, + }); + expect(tooHigh.isErr()).toBe(true); + if (tooHigh.isErr()) expect(tooHigh.error.type).toBe("invalid_override"); + + const tooLow = await system.queues.overrideQueueConcurrencyLimit(authEnv, queue.friendlyId, { + percent: 0, + }); + expect(tooLow.isErr()).toBe(true); + if (tooLow.isErr()) expect(tooLow.error.type).toBe("invalid_override"); + + const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } }); + expect(row.concurrencyLimitOverriddenAt).toBeNull(); + }); + + postgresTest("reset clears both the absolute limit and the percent", async ({ prisma }) => { + const { queue, authEnv } = await seedEnvAndQueue(prisma, { + maximumConcurrencyLimit: 10, + queueConcurrencyLimit: 8, + }); + + const system = new ConcurrencySystem({ db: prisma, reader: prisma }); + + const overridden = await system.queues.overrideQueueConcurrencyLimit( + authEnv, + queue.friendlyId, + { percent: 50 } + ); + expect(overridden.isOk()).toBe(true); + + const reset = await system.queues.resetConcurrencyLimit(authEnv, queue.friendlyId); + expect(reset.isOk()).toBe(true); + + const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } }); + // restored to the base captured at override time + expect(row.concurrencyLimit).toBe(8); + expect(row.concurrencyLimitOverridePercent).toBeNull(); + expect(row.concurrencyLimitBase).toBeNull(); + expect(row.concurrencyLimitOverriddenAt).toBeNull(); + expect(row.concurrencyLimitOverriddenBy).toBeNull(); + }); + + postgresTest( + "recalculatePercentLimits recomputes percent queues and leaves absolute overrides untouched", + async ({ prisma }) => { + const { project, environment, authEnv } = await seedEnvAndQueue(prisma, { + maximumConcurrencyLimit: 10, + queueConcurrencyLimit: 8, + }); + + const system = new ConcurrencySystem({ db: prisma, reader: prisma }); + + // A percent-based queue: 50% of env(10) -> 5 + const percentQueue = await prisma.taskQueue.create({ + data: { + friendlyId: `queue_pct_${Math.random().toString(36).slice(2, 8)}`, + name: `task/pct-${Math.random().toString(36).slice(2, 8)}`, + projectId: project.id, + runtimeEnvironmentId: environment.id, + concurrencyLimit: 6, + }, + }); + const pctResult = await system.queues.overrideQueueConcurrencyLimit( + authEnv, + percentQueue.friendlyId, + { percent: 50 } + ); + expect(pctResult.isOk()).toBe(true); + { + const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: percentQueue.id } }); + expect(row.concurrencyLimit).toBe(5); // floor(10 * 0.5) + } + + // An absolute-override queue: limit 4, no percent + const absQueue = await prisma.taskQueue.create({ + data: { + friendlyId: `queue_abs_${Math.random().toString(36).slice(2, 8)}`, + name: `task/abs-${Math.random().toString(36).slice(2, 8)}`, + projectId: project.id, + runtimeEnvironmentId: environment.id, + concurrencyLimit: 6, + }, + }); + const absResult = await system.queues.overrideQueueConcurrencyLimit( + authEnv, + absQueue.friendlyId, + { limit: 4 } + ); + expect(absResult.isOk()).toBe(true); + + // Simulate the environment limit changing from 10 -> 20 and recalculate. + const bumpedEnv = { + id: environment.id, + maximumConcurrencyLimit: 20, + } as unknown as AuthenticatedEnvironment; + + const outcome = await system.queues.recalculatePercentLimits(bumpedEnv); + expect(outcome.total).toBe(1); // only the percent queue is considered + expect(outcome.updated).toBe(1); + + const pctRow = await prisma.taskQueue.findUniqueOrThrow({ where: { id: percentQueue.id } }); + // floor(20 * 50 / 100) = 10 + expect(pctRow.concurrencyLimit).toBe(10); + expect(pctRow.concurrencyLimitOverridePercent?.toNumber()).toBe(50); + + const absRow = await prisma.taskQueue.findUniqueOrThrow({ where: { id: absQueue.id } }); + // absolute override must be untouched + expect(absRow.concurrencyLimit).toBe(4); + expect(absRow.concurrencyLimitOverridePercent).toBeNull(); + } + ); + + postgresTest( + "recalculatePercentLimits is idempotent when the limit is unchanged", + async ({ prisma }) => { + const { queue, authEnv } = await seedEnvAndQueue(prisma, { + maximumConcurrencyLimit: 10, + queueConcurrencyLimit: 8, + }); + + const system = new ConcurrencySystem({ db: prisma, reader: prisma }); + + const overridden = await system.queues.overrideQueueConcurrencyLimit( + authEnv, + queue.friendlyId, + { percent: 50 } + ); + expect(overridden.isOk()).toBe(true); + + // Recalculate against the SAME environment limit -> nothing to update. + const outcome = await system.queues.recalculatePercentLimits(authEnv); + expect(outcome.total).toBe(1); + expect(outcome.updated).toBe(0); + + const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } }); + expect(row.concurrencyLimit).toBe(5); + } + ); + + postgresTest( + "recalculatePercentLimits re-syncs the engine on a later recalc after an engine push failed, even when the DB limit is unchanged", + async ({ prisma }) => { + const { queue, authEnv } = await seedEnvAndQueue(prisma, { + maximumConcurrencyLimit: 10, + queueConcurrencyLimit: 8, + }); + + const system = new ConcurrencySystem({ db: prisma, reader: prisma }); + + const overridden = await system.queues.overrideQueueConcurrencyLimit( + authEnv, + queue.friendlyId, + { percent: 50 } + ); + expect(overridden.isOk()).toBe(true); + { + const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } }); + expect(row.concurrencyLimit).toBe(5); // floor(10 * 0.5) + } + + const bumpedEnv = { + id: authEnv.id, + maximumConcurrencyLimit: 20, + } as unknown as AuthenticatedEnvironment; + + // The env-limit bump recalc writes the new DB limit (10) but the engine push fails and is + // swallowed by the per-queue catch: DB and engine are now diverged (DB=10, engine=5). + updateQueueConcurrencyLimitsMock.mockClear(); + updateQueueConcurrencyLimitsMock.mockRejectedValueOnce(new Error("engine push failed")); + + const first = await system.queues.recalculatePercentLimits(bumpedEnv); + expect(first.updated).toBe(1); // DB was written before the push threw + { + const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } }); + expect(row.concurrencyLimit).toBe(10); + } + + // A later recalc at the SAME env limit makes no DB change, but MUST still push to the engine + // so the previously-failed sync self-heals. (Regression guard: the old code `continue`d when + // newLimit === concurrencyLimit and left the engine stuck at 5 forever.) + updateQueueConcurrencyLimitsMock.mockClear(); + const second = await system.queues.recalculatePercentLimits(bumpedEnv); + expect(second.updated).toBe(0); // no DB write + expect(updateQueueConcurrencyLimitsMock).toHaveBeenCalledWith( + expect.anything(), + queue.name, + 10 + ); + } + ); +}); diff --git a/apps/webapp/test/queueMetricsMapping.test.ts b/apps/webapp/test/queueMetricsMapping.test.ts new file mode 100644 index 00000000000..61e3893c7fb --- /dev/null +++ b/apps/webapp/test/queueMetricsMapping.test.ts @@ -0,0 +1,239 @@ +import { describe, expect, it } from "vitest"; +import { + descriptorFromQueue, + mapEntryToRows, + OVERFLOW_QUEUE_NAME, + QueueNameLimiter, +} from "~/v3/queueMetricsMapping"; + +describe("descriptorFromQueue", () => { + it("parses a plain descriptor", () => { + expect(descriptorFromQueue("{org:o1}:proj:p1:env:e1:queue:task/my-task")).toEqual({ + organization_id: "o1", + project_id: "p1", + environment_id: "e1", + queue_name: "task/my-task", + concurrency_key: "", + }); + }); + + it("captures a concurrency-key suffix", () => { + expect(descriptorFromQueue("{org:o1}:proj:p1:env:e1:queue:task/t:ck:tenant-3")).toEqual( + expect.objectContaining({ queue_name: "task/t", concurrency_key: "tenant-3" }) + ); + }); + + it("maps the ck wildcard to no key", () => { + expect(descriptorFromQueue("{org:o1}:proj:p1:env:e1:queue:task/t:ck:*")).toEqual( + expect.objectContaining({ queue_name: "task/t", concurrency_key: "" }) + ); + }); + + it("keeps colons inside the queue name", () => { + expect(descriptorFromQueue("{org:o1}:proj:p1:env:e1:queue:my:odd:queue")).toEqual( + expect.objectContaining({ queue_name: "my:odd:queue", concurrency_key: "" }) + ); + }); + + it("keeps colons in the name while capturing a real ck suffix", () => { + expect(descriptorFromQueue("{org:o1}:proj:p1:env:e1:queue:a:b:ck:t9")).toEqual( + expect.objectContaining({ queue_name: "a:b", concurrency_key: "t9" }) + ); + }); + + it("rejects malformed descriptors", () => { + expect(descriptorFromQueue("not-a-descriptor")).toBeNull(); + expect(descriptorFromQueue("{org:o1}:proj:p1:env:e1")).toBeNull(); + expect(descriptorFromQueue("")).toBeNull(); + }); +}); + +describe("QueueNameLimiter", () => { + it("passes names through under the cap and overflows past it, per scope", () => { + const limiter = new QueueNameLimiter(2); + expect(limiter.limit("env1", "a")).toBe("a"); + expect(limiter.limit("env1", "b")).toBe("b"); + expect(limiter.limit("env1", "c")).toBe(OVERFLOW_QUEUE_NAME); + expect(limiter.limit("env1", "a")).toBe("a"); + expect(limiter.limit("env2", "c")).toBe("c"); + }); + + it("is unlimited when the cap is 0", () => { + const limiter = new QueueNameLimiter(0); + for (let i = 0; i < 100; i++) { + expect(limiter.limit("env1", `q${i}`)).toBe(`q${i}`); + } + }); + + it("evicts the oldest scope when the scope map is full", () => { + const limiter = new QueueNameLimiter(1, 2); + expect(limiter.limit("env1", "a")).toBe("a"); + expect(limiter.limit("env2", "a")).toBe("a"); + expect(limiter.limit("env3", "a")).toBe("a"); + expect(limiter.limit("env1", "b")).toBe("b"); + }); +}); + +describe("mapEntryToRows", () => { + const q = "{org:o1}:proj:p1:env:e1:queue:task/t"; + + it("maps a gauge entry with numeric fields", () => { + const rows = mapEntryToRows({ + id: "1700000000000-0", + fields: { + op: "gauge", + q, + ql: "5", + cc: "2", + lim: "10", + eql: "7", + ec: "3", + elim: "20", + thr: "1", + }, + }); + expect(rows).toHaveLength(1); + expect(rows[0]).toEqual( + expect.objectContaining({ + op: "gauge", + organization_id: "o1", + queue_name: "task/t", + concurrency_key: "", + queued: 5, + running: 2, + queue_limit: 10, + env_queued: 7, + env_running: 3, + env_limit: 20, + throttled: 1, + }) + ); + expect(rows[0]!.event_time).toBe("2023-11-14 22:13:20"); + expect(rows[0]!.ck_backlogged).toBeUndefined(); + expect(rows[0]!.ck_max_wait_ms).toBeUndefined(); + }); + + it("keeps the key on per-subqueue gauges and maps the CK-health tail", () => { + const rows = mapEntryToRows({ + id: "1700000000000-0", + fields: { op: "gauge", q: `${q}:ck:tenant-1`, ql: "4", ckq: "3", ckw: "2500" }, + }); + expect(rows).toHaveLength(1); + expect(rows[0]).toEqual( + expect.objectContaining({ + op: "gauge", + queue_name: "task/t", + concurrency_key: "tenant-1", + queued: 4, + ck_backlogged: 3, + ck_max_wait_ms: 2500, + }) + ); + }); + + it("maps started with wait_ms + cumulative and drops unknown ops", () => { + const started = mapEntryToRows({ + id: "1700000000000-0", + fields: { op: "started", q, wait: "48", cum: "512" }, + }); + expect(started).toHaveLength(1); + expect(started[0]).toEqual( + expect.objectContaining({ + op: "started", + wait_ms: 48, + cumulative: 512, + order_key: (1700000000000n * 1000000n).toString(), + }) + ); + expect(mapEntryToRows({ id: "1-0", fields: { op: "ack", q, cum: "9" } })[0]).toEqual( + expect.objectContaining({ op: "ack", cumulative: 9 }) + ); + expect(mapEntryToRows({ id: "1-0", fields: { op: "bogus", q } })).toEqual([]); + expect(mapEntryToRows({ id: "1-0", fields: { op: "ack" } })).toEqual([]); + }); + + it("expands a dual-odometer counter entry into base + per-key rows", () => { + const rows = mapEntryToRows({ + id: "1700000000000-3", + fields: { op: "started", q, ck: "tenant-9", wait: "80", cum: "41", ckcum: "7" }, + }); + expect(rows).toHaveLength(2); + expect(rows[0]).toEqual( + expect.objectContaining({ queue_name: "task/t", cumulative: 41, wait_ms: 80 }) + ); + expect(rows[0]!.concurrency_key).toBeUndefined(); + expect(rows[1]).toEqual( + expect.objectContaining({ + queue_name: "task/t", + concurrency_key: "tenant-9", + cumulative: 7, + wait_ms: 80, + }) + ); + expect(rows[0]!.order_key).toBe(rows[1]!.order_key); + + // Baseline entries carry exactly one odometer each. + const baseBaseline = mapEntryToRows({ id: "1-0", fields: { op: "started", q, cum: "0" } }); + expect(baseBaseline).toHaveLength(1); + expect(baseBaseline[0]!.concurrency_key).toBeUndefined(); + const ckBaseline = mapEntryToRows({ + id: "1-1", + fields: { op: "started", q, ck: "tenant-9", ckcum: "0" }, + }); + expect(ckBaseline).toHaveLength(1); + expect(ckBaseline[0]).toEqual( + expect.objectContaining({ concurrency_key: "tenant-9", cumulative: 0 }) + ); + }); + + it("applies the queue-name limiter: gauges overflow, counters drop", () => { + const limiters = { queueNames: new QueueNameLimiter(1) }; + const first = mapEntryToRows({ id: "1-0", fields: { op: "ack", q, cum: "1" } }, limiters); + expect(first[0]!.queue_name).toBe("task/t"); + + // Overflowed gauges keep flowing under the shared name (max stays meaningful), + // with per-key attribution stripped. + const overflowGauge = mapEntryToRows( + { + id: "1-1", + fields: { op: "gauge", q: "{org:o1}:proj:p1:env:e1:queue:task/other:ck:t1", ql: "3" }, + }, + limiters + ); + expect(overflowGauge[0]!.queue_name).toBe(OVERFLOW_QUEUE_NAME); + expect(overflowGauge[0]!.concurrency_key).toBe(""); + + // Overflowed counters are dropped: merging distinct odometers under one key + // produces garbage deltas. + const overflowCounter = mapEntryToRows( + { id: "1-2", fields: { op: "ack", q: "{org:o1}:proj:p1:env:e1:queue:task/other", cum: "4" } }, + limiters + ); + expect(overflowCounter).toEqual([]); + }); + + it("applies the concurrency-key limiter: overflow drops the per-key row, keeps base", () => { + const limiters = { concurrencyKeys: new QueueNameLimiter(1) }; + const first = mapEntryToRows( + { id: "1-0", fields: { op: "ack", q, ck: "t1", cum: "5", ckcum: "2" } }, + limiters + ); + expect(first).toHaveLength(2); + + const overflowed = mapEntryToRows( + { id: "1-1", fields: { op: "ack", q, ck: "t2", cum: "6", ckcum: "1" } }, + limiters + ); + expect(overflowed).toHaveLength(1); + expect(overflowed[0]!.cumulative).toBe(6); + expect(overflowed[0]!.concurrency_key).toBeUndefined(); + + // Gauge for an overflowed key keeps the row but loses the attribution. + const overflowGauge = mapEntryToRows( + { id: "1-2", fields: { op: "gauge", q: `${q}:ck:t3`, ql: "2" } }, + limiters + ); + expect(overflowGauge).toHaveLength(1); + expect(overflowGauge[0]!.concurrency_key).toBe(""); + }); +}); diff --git a/apps/webapp/test/queueSparklineGrid.test.ts b/apps/webapp/test/queueSparklineGrid.test.ts new file mode 100644 index 00000000000..ac97ccb6901 --- /dev/null +++ b/apps/webapp/test/queueSparklineGrid.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; +import { bucketIndex, computeSparklineGrid, SPARKLINE_POINTS } from "~/v3/queueSparklineGrid"; + +const PERIODS_MS = { + "30m": 30 * 60 * 1000, + "1h": 60 * 60 * 1000, + "1d": 24 * 60 * 60 * 1000, + "7d": 7 * 24 * 60 * 60 * 1000, + "30d": 30 * 24 * 60 * 60 * 1000, +}; + +/** Newest bucket the ClickHouse query can return: the last full slot before the grid end. */ +function newestBucketMs(grid: ReturnType<typeof computeSparklineGrid>): number { + return grid.endMs - grid.bucketIntervalMs; +} + +describe("computeSparklineGrid", () => { + it("keeps the newest bucket for every period, at any start offset", () => { + for (const [label, span] of Object.entries(PERIODS_MS)) { + for (let offset = 0; offset < 120; offset++) { + const toMs = 1785000000000 + offset * 1371; + const grid = computeSparklineGrid(new Date(toMs - span), new Date(toMs)); + + expect( + bucketIndex(grid, newestBucketMs(grid)), + `${label} dropped its newest bucket at offset ${offset}` + ).not.toBeNull(); + } + } + }); + + it("indexes every bucket the query can return, and nothing outside the grid", () => { + const toMs = 1785000000123; + const grid = computeSparklineGrid(new Date(toMs - PERIODS_MS["1d"]), new Date(toMs)); + + expect(bucketIndex(grid, grid.bucketStartMs)).toBe(0); + expect(bucketIndex(grid, newestBucketMs(grid))).toBe(grid.bucketCount - 1); + expect(bucketIndex(grid, grid.bucketStartMs - grid.bucketIntervalMs)).toBeNull(); + expect(bucketIndex(grid, grid.endMs)).toBeNull(); + }); + + it("aligns the grid outward from the requested range", () => { + const from = new Date(1785000000123); + const to = new Date(from.getTime() + PERIODS_MS["1h"]); + const grid = computeSparklineGrid(from, to); + + expect(grid.bucketStartMs).toBeLessThanOrEqual(from.getTime()); + expect(grid.endMs).toBeGreaterThanOrEqual(to.getTime()); + expect(grid.bucketStartMs % grid.bucketIntervalMs).toBe(0); + expect(grid.endMs % grid.bucketIntervalMs).toBe(0); + }); + + it("targets the sparkline resolution without going below a one minute bucket", () => { + const toMs = 1785000000000; + const dayGrid = computeSparklineGrid(new Date(toMs - PERIODS_MS["1d"]), new Date(toMs)); + expect(dayGrid.bucketSeconds).toBe(1800); + expect(dayGrid.bucketCount).toBeGreaterThanOrEqual(SPARKLINE_POINTS); + + const shortGrid = computeSparklineGrid(new Date(toMs - 60_000), new Date(toMs)); + expect(shortGrid.bucketSeconds).toBe(60); + expect(shortGrid.bucketCount).toBeGreaterThanOrEqual(1); + }); + + it("never returns a zero-bucket grid for a degenerate range", () => { + const at = new Date(1785000000000); + const grid = computeSparklineGrid(at, at); + expect(grid.bucketCount).toBeGreaterThanOrEqual(1); + expect(grid.bucketSeconds).toBe(60); + }); +}); diff --git a/apps/webapp/test/reportHealth.test.ts b/apps/webapp/test/reportHealth.test.ts new file mode 100644 index 00000000000..22f940ffbdd --- /dev/null +++ b/apps/webapp/test/reportHealth.test.ts @@ -0,0 +1,511 @@ +import { describe, expect, it } from "vitest"; +import { renderReportAnsi, renderReportMarkdown } from "~/presenters/v3/reports/renderMarkdown"; +import { + interpret, + isPendingIncreasing, + type HealthInput, +} from "~/presenters/v3/reports/health/health"; + +/** Golden A — degraded: env concurrency-limit saturation, backlog drains. */ +const INPUT_A: HealthInput = { + scope: "prod", + period: "last 1h", + baselineLabel: "vs your 7d normal", + generatedAt: "2026-07-20T12:00:00.000Z", + windowMinutes: 60, + flowSource: "queue_metrics_v1", + pending: { now: 1910, normal: 120, series: [120, 300, 700, 1200, 1600, 1910], estimated: false }, + startLatency: { + p95Ms: 42000, + normalP95Ms: 7000, + series: [7000, 12000, 20000, 30000, 38000, 42000], + }, + throughput: { donePerMin: 820, triggeredPerMin: 1150, normalTriggeredPerMin: 1100 }, + failures: { rate: 0.013, normalRate: 0.011, series: [0.011, 0.011, 0.012, 0.013] }, + duration: { p95Ms: 1200, normalP95Ms: 1180 }, + liveness: { telemetryAgeMs: 4000 }, + flowEvidence: { + runningSeries: [60, 80, 90, 100, 100, 100, 100, 100, 100], + envLimit: 100, + throttledShare: 0.1, + worstQueue: { name: "email-sends", share: 0.82 }, + dlqDelta: 0, + }, +}; + +/** Golden B — everything healthy. */ +const INPUT_B: HealthInput = { + scope: "prod", + period: "last 1h", + baselineLabel: "vs your 7d normal", + generatedAt: "2026-07-20T12:00:00.000Z", + windowMinutes: 60, + flowSource: "queue_metrics_v1", + pending: { now: 84, normal: 120, series: [110, 96, 88, 90, 84], estimated: false }, + startLatency: { p95Ms: 6000, normalP95Ms: 7000, series: [6500, 6200, 6000, 5900, 6000] }, + throughput: { donePerMin: 1000, triggeredPerMin: 1000, normalTriggeredPerMin: 1000 }, + failures: { rate: 0.009, normalRate: 0.011, series: [0.01, 0.009, 0.009] }, + duration: { p95Ms: 1100, normalP95Ms: 1180 }, + liveness: { telemetryAgeMs: 2000 }, + flowEvidence: { + runningSeries: [40, 45, 50, 48, 44], + envLimit: 100, + throttledShare: 0, + worstQueue: null, + dlqDelta: 0, + }, +}; + +describe("health cause tree (Golden A — env limit saturation)", () => { + const vm = interpret(INPUT_A); + const flow = vm.findings.find((f) => f.type === "flow")!; + + it("selects the env-limit-saturation cause + evidence", () => { + expect(flow.reason).toBe("env_limit_saturation"); + expect(flow.severity).toBe("warn"); // crit raw, warn by the drainable policy + expect(flow.anomalyWindow).toEqual({ minutes: 40, touchesEnd: true }); + expect(flow.attribution).toEqual({ + dim: "queue", + key: "email-sends", + share: 0.82, + of: "pending", + }); + expect(flow.exclusions).toEqual([]); // env-limit saturation rules nothing out... + expect(flow.observations).toEqual([ + // ...it states supporting facts instead. + { code: "not_workers_platform", evidence: { donePerMin: 820 } }, + { code: "nothing_dead_lettered", evidence: { dlq: 0 } }, + ]); + expect(flow.read).toBe("saturation_chain"); + const concurrency = vm.metrics.find((m) => m.id === "concurrency")!; + expect(concurrency.annotation).toEqual({ code: "pinned_minutes", value: 40 }); + }); + + it("footer = raise limit + do-nothing (drains)", () => { + expect(vm.footer).toEqual([ + { code: "raise_env_limit", link: "concurrency" }, + { code: "do_nothing_drains", value: 2.3 }, + ]); + }); + + it("renders", () => { + expect(renderReportMarkdown(vm)).toMatchInlineSnapshot(` + "/report health prod · last 1h · vs your 7d normal + + 🟡 Flow slowing · 🟢 Execution healthy · 🟢 data fresh + + FLOW 🟡 at your env concurrency limit (last 40 min) + + concurrency 100/100 ▁▅▆█████ pinned 40 of last 60 min + + pending 1,910 ↑ 16× ▁▁▂▃▅▅▇█ (normal ~120) + + start latency p95 42s ↑ 6× ▁▁▂▄▆▆▇█ (normal ~7s) + + worst queue email-sends — 82% of pending + + read: limit saturated → incoming work exceeds capacity → backlog grows + runs are completing at ~820/min + nothing dead-lettered + + EXECUTION 🟢 the runs that DO start are fine + + failures 1.3% (normal ~1.1%) · durations normal + read: runs are completing normally + + LIVENESS 🟢 fresh — telemetry current, updated 4s ago + + → Raise the env concurrency limit + or do nothing — backlog drains in ~2.3 min once triggers ease" + `); + }); +}); + +describe("health (Golden B — healthy)", () => { + const vm = interpret(INPUT_B); + + it("all findings healthy, footer nothing-to-do", () => { + expect(vm.summary.severity).toBe("ok"); + expect(vm.findings.map((f) => f.severity)).toEqual(["ok", "ok", "ok"]); + expect(vm.footer).toEqual([{ code: "nothing_to_do" }]); + }); + + it("renders", () => { + expect(renderReportMarkdown(vm)).toMatchInlineSnapshot(` + "/report health prod · last 1h · vs your 7d normal + + 🟢 Flow healthy · 🟢 Execution healthy · 🟢 data fresh + + FLOW 🟢 starting normally — pending 84 (normal ~120) · starts p95 6s + + EXECUTION 🟢 completing normally — failures 0.9% (normal ~1.1%) · durations normal + + LIVENESS 🟢 fresh — telemetry current, updated 2s ago + + → nothing to do" + `); + }); +}); + +describe("snapshot fallback path flags the estimated backlog trend", () => { + // No queue-metrics evidence -> the backlog series is an estimate (finished-vs-triggered proxy). + const snapshot: HealthInput = { + ...INPUT_B, + flowSource: "snapshot+runs", + pending: { now: 6000, series: [1000, 3000, 6000], estimated: true }, // normal omitted on snapshot + flowEvidence: { + runningSeries: [], + envLimit: 0, + throttledShare: 0, + worstQueue: null, + dlqDelta: null, + }, + }; + + it("renders an (estimated) caveat on the proxy series, not a bare sparkline", () => { + const vm = interpret(snapshot); + expect(vm.findings.find((f) => f.type === "flow")!.reason).toBe("backlog"); + const md = renderReportMarkdown(vm); + expect(md).toContain("(estimated)"); // the human surface signals the trend is a proxy + }); +}); + +describe("liveness trust guard (telemetry freshness)", () => { + it("stale telemetry forces execution unknown + crit summary + control-plane footer", () => { + const stale: HealthInput = { ...INPUT_A, liveness: { telemetryAgeMs: 600_000 } }; + const vm = interpret(stale); + const execution = vm.findings.find((f) => f.type === "execution")!; + expect(execution.reason).toBe("unknown"); + expect(execution.read).toBe("data_stale"); + expect(vm.summary.severity).toBe("crit"); + // #6: footer points at the pipeline, not "raise the env limit" off stale data. + expect(vm.footer).toEqual([{ code: "check_control_plane", link: "status" }]); + }); + + it("no freshness signal is 'unknown', NOT stale — it does not trust-guard execution", () => { + const unknown: HealthInput = { + ...INPUT_A, + // healthy execution so we can see the guard did NOT fire. + failures: { rate: 0.009, normalRate: 0.011, series: [0.009] }, + liveness: { telemetryAgeMs: null }, + }; + const vm = interpret(unknown); + const execution = vm.findings.find((f) => f.type === "execution")!; + const liveness = vm.findings.find((f) => f.type === "liveness")!; + expect(liveness.reason).toBe("freshness_unknown"); + expect(liveness.severity).toBe("ok"); // no signal is NEUTRAL, not a warning + expect(execution.reason).not.toBe("unknown"); + }); + + it("a healthy but idle env (no telemetry signal) reads overall green, not yellow", () => { + // Golden B is all-healthy; drop its telemetry signal -> the verdict must stay ok, since + // "freshness unknown" is neutral and must not drag a fine env into a yellow report. + const vm = interpret({ ...INPUT_B, liveness: { telemetryAgeMs: null } }); + expect(vm.summary.severity).toBe("ok"); + expect(vm.findings.find((f) => f.type === "liveness")!.reason).toBe("freshness_unknown"); + // ...but the marker is NEUTRAL (⚪), not a confident green — the state is genuinely unknown. + const md = renderReportMarkdown(vm); + expect(md).toContain("⚪"); + expect(md).not.toContain("🟡"); + }); +}); + +describe("isPendingIncreasing", () => { + it("detects a positive trend", () => { + expect(isPendingIncreasing([120, 300, 700, 1200, 1600, 1910])).toBe(true); + expect(isPendingIncreasing([110, 96, 88, 90, 84])).toBe(false); + }); +}); + +/** + * Fixed-priority cause tree: the first discriminator that fires wins. Each case starts + * from Golden A and overrides ONLY flow evidence so the intended discriminator matches + * (Golden A itself covers env_limit_saturation). + */ +describe("flow cause tree — cause selection per discriminator", () => { + const withFlow = ( + flowEvidence: Partial<HealthInput["flowEvidence"]>, + throughput?: Partial<HealthInput["throughput"]> + ): HealthInput => ({ + ...INPUT_A, + throughput: { ...INPUT_A.throughput, ...throughput }, + flowEvidence: { ...INPUT_A.flowEvidence, ...flowEvidence }, + }); + + const flowReason = (input: HealthInput) => + interpret(input).findings.find((f) => f.type === "flow")!.reason; + + it("env_limit_saturation — concurrency pinned at the env limit", () => { + expect(flowReason(INPUT_A)).toBe("env_limit_saturation"); + }); + + it("dequeue_stall — capacity idle while the backlog grows", () => { + // running far below the limit (0.1) with a rising backlog + elevated latency. + expect(flowReason(withFlow({ runningSeries: Array(9).fill(10) }))).toBe("dequeue_stall"); + }); + + it("queue_limit_throttling — throttled, but the env limit is not the bottleneck", () => { + expect(flowReason(withFlow({ runningSeries: Array(9).fill(50), throttledShare: 0.5 }))).toBe( + "queue_limit_throttling" + ); + }); + + it("selects queue throttling over dequeue stall when both shapes match", () => { + // low running (would look like a stall) BUT the queue is throttling — the known config + // bottleneck must win, not "it's on our side". + expect(flowReason(withFlow({ runningSeries: Array(9).fill(10), throttledShare: 0.5 }))).toBe( + "queue_limit_throttling" + ); + }); + + it("trigger_spike — triggers >= 3x the normal rate", () => { + expect( + flowReason( + withFlow( + { runningSeries: Array(9).fill(50), throttledShare: 0 }, + { triggeredPerMin: 3300, normalTriggeredPerMin: 1100 } + ) + ) + ).toBe("trigger_spike"); + }); + + it("trigger_surge — new volume with no baseline (multiplier can't be computed)", () => { + // normal 0 makes a multiplier meaningless, so an absolute rate selects "new volume" + // instead of dropping to the v1 fallback (a spike from a zero baseline was invisible before). + const input = withFlow( + { runningSeries: Array(9).fill(50), throttledShare: 0 }, + { triggeredPerMin: 5000, normalTriggeredPerMin: 0 } + ); + expect(flowReason(input)).toBe("trigger_surge"); + const triggered = interpret(input).metrics.find((m) => m.id === "triggered")!; + expect(triggered.annotation).toEqual({ code: "surge_rate", value: 5000 }); + }); + + it("does not select trigger_spike when completions keep pace and pending falls", () => { + // 3× the normal trigger rate, but the backlog is draining (net >= 0, pending falling) — so the + // spike is NOT the cause of degradation (elevated latency is). Blaming it would contradict + // its own "queue fills faster than it drains" read. + const input: HealthInput = { + ...INPUT_A, + pending: { now: 400, normal: 1000, series: [500, 450, 400], estimated: false }, + throughput: { donePerMin: 3300, triggeredPerMin: 3300, normalTriggeredPerMin: 1100 }, + flowEvidence: { + ...INPUT_A.flowEvidence, + runningSeries: Array(9).fill(50), + throttledShare: 0, + }, + }; + expect(flowReason(input)).not.toBe("trigger_spike"); + expect(flowReason(input)).toBe("start_latency"); // falls through to the v1 symptom + }); + + it("does not select trigger_surge when new volume is draining", () => { + // No baseline + high volume, but completions outpace triggers and the backlog falls — not a backup. + const input: HealthInput = { + ...INPUT_A, + pending: { now: 400, normal: 1000, series: [500, 450, 400], estimated: false }, + throughput: { donePerMin: 6000, triggeredPerMin: 5000, normalTriggeredPerMin: 0 }, + flowEvidence: { + ...INPUT_A.flowEvidence, + runningSeries: Array(9).fill(50), + throttledShare: 0, + }, + }; + expect(flowReason(input)).not.toBe("trigger_surge"); + }); + + it("fallback — degraded with no discriminator -> v1 symptom (start latency)", () => { + expect(flowReason(withFlow({ runningSeries: Array(9).fill(50), throttledShare: 0 }))).toBe( + "start_latency" + ); + }); +}); + +describe("env_limit_saturation read does not claim a start lag that isn't there", () => { + // Pinned concurrency + rising backlog, but start latency is still healthy — saturation can grow + // a backlog before p95 latency crosses its threshold, so the read must not assert "starts lag". + const input: HealthInput = { + ...INPUT_A, + startLatency: { p95Ms: 6000, normalP95Ms: 7000, series: [6000, 6100, 6000, 5900, 6000] }, + flowEvidence: { ...INPUT_A.flowEvidence, runningSeries: Array(9).fill(100) }, + }; + const vm = interpret(input); + const flow = vm.findings.find((f) => f.type === "flow")!; + + it("still selects env_limit_saturation with a latency-free read", () => { + expect(flow.reason).toBe("env_limit_saturation"); + expect(flow.read).toBe("saturation_chain"); + expect(vm.metrics.find((m) => m.id === "start_latency_p95")!.severity).toBe("ok"); + const md = renderReportMarkdown(vm); + expect(md).toContain("incoming work exceeds capacity"); + expect(md).not.toContain("starts lag"); + }); +}); + +describe("trigger spike does not exonerate user code", () => { + const spike = interpret({ + ...INPUT_A, + throughput: { donePerMin: 820, triggeredPerMin: 3300, normalTriggeredPerMin: 1100 }, + flowEvidence: { ...INPUT_A.flowEvidence, runningSeries: Array(9).fill(50), throttledShare: 0 }, + }); + + it("reports execution is healthy but never claims 'NOT a code problem'", () => { + expect(spike.findings.find((f) => f.type === "flow")!.reason).toBe("trigger_spike"); + const md = renderReportMarkdown(spike); + expect(md).toContain("runs that start are completing normally"); // flow exclusion (proven fact) + expect(md).not.toContain("NOT a code problem"); // a code path may BE flooding the queue + }); + + it("dequeue_stall (platform-side) still reads 'NOT a code problem'", () => { + const stall = interpret({ + ...INPUT_A, + flowEvidence: { ...INPUT_A.flowEvidence, runningSeries: Array(9).fill(10) }, + }); + expect(stall.findings.find((f) => f.type === "flow")!.reason).toBe("dequeue_stall"); + expect(renderReportMarkdown(stall)).toContain("NOT a code problem"); + }); +}); + +describe("ANSI render (terminal)", () => { + const ansi = renderReportAnsi(interpret(INPUT_A)); + + it("uses glyphs + ANSI colour, never the markdown status emoji", () => { + expect(ansi).toMatch(/\x1b\[\d+m/); // an ANSI SGR colour code + expect(ansi).toMatch(/[✓⚠✕]/); // severity glyphs (not emoji) + expect(ansi).not.toMatch(/[🟢🟡🔴]/u); // emoji are the markdown surface only + }); +}); + +describe("exclusions are proven, not assumed", () => { + const withFlow = ( + flowEvidence: Partial<HealthInput["flowEvidence"]>, + over: Partial<HealthInput> = {} + ): HealthInput => ({ + ...INPUT_A, + ...over, + flowEvidence: { ...INPUT_A.flowEvidence, ...flowEvidence }, + }); + const exclusionCodes = (input: HealthInput) => + (interpret(input).findings.find((f) => f.type === "flow")!.exclusions ?? []).map((e) => e.code); + const observationCodes = (input: HealthInput) => + (interpret(input).findings.find((f) => f.type === "flow")!.observations ?? []).map( + (o) => o.code + ); + + it("dequeue_stall claims not-your-code AND not-your-config (both proven: healthy exec, no pin, no throttle)", () => { + // dequeue_stall only fires with no env-pin and no throttling, so "limits aren't the + // bottleneck" is genuinely proven here (a throttled shape selects queue_limit_throttling). + const codes = exclusionCodes(withFlow({ runningSeries: Array(9).fill(10) })); + expect(codes).toContain("not_your_code"); + expect(codes).toContain("not_your_config"); + }); + + it("trigger_spike observes healthy execution without ruling out user code", () => { + // Backing-up spike (net < 0, pending rising) with healthy execution -> "execution_healthy" as + // an OBSERVATION, NOT the exclusion "not_your_code": a code path fanning out task.trigger could + // BE the cause of the spike, so it must not be ruled out. + const healthyInput = withFlow( + { runningSeries: Array(9).fill(50), throttledShare: 0 }, + { throughput: { donePerMin: 820, triggeredPerMin: 3300, normalTriggeredPerMin: 1100 } } + ); + expect(observationCodes(healthyInput)).toContain("execution_healthy"); + expect(exclusionCodes(healthyInput)).not.toContain("not_your_code"); + + // Execution failing -> can't even observe that execution is healthy. + const degradedInput = withFlow( + { runningSeries: Array(9).fill(50), throttledShare: 0 }, + { + throughput: { donePerMin: 820, triggeredPerMin: 3300, normalTriggeredPerMin: 1100 }, + failures: { rate: 0.2, normalRate: 0.01, series: [0.2] }, + } + ); + expect(observationCodes(degradedInput)).not.toContain("execution_healthy"); + }); +}); + +describe("stale-telemetry trust guard covers flow (not just execution)", () => { + const stale = interpret({ ...INPUT_A, liveness: { telemetryAgeMs: 600_000 } }); + const flow = stale.findings.find((f) => f.type === "flow")!; + const execution = stale.findings.find((f) => f.type === "execution")!; + + it("marks flow unknown + crit and strips its action / attribution / exclusions / anomaly window", () => { + expect(flow.reason).toBe("unknown"); + expect(flow.severity).toBe("crit"); // consistent across summary / section glyph / JSON + expect(flow.recommendation).toBeUndefined(); + expect(flow.attribution).toBeUndefined(); + expect(flow.exclusions).toBeUndefined(); + expect(flow.observations).toBeUndefined(); + expect(flow.anomalyWindow).toBeUndefined(); // no stale causal evidence left in the VM + }); + + it("marks execution unknown + crit too", () => { + expect(execution.reason).toBe("unknown"); + expect(execution.severity).toBe("crit"); + }); + + it("strips stale-derived metric annotations so format=json can't leak them", () => { + // Golden A sets concurrency.annotation ("pinned 40 of last 60 min") before the guard runs; + // a stale feed must not surface that narrative on the raw JSON metrics. + expect(stale.metrics.every((m) => m.annotation === undefined)).toBe(true); + }); + + it("renders both sections red as unknown, with no stale causal verdict", () => { + // The unknown headline already says "data stale"; there's no `read:` line to render (it would + // just repeat that), and no stale causal evidence (anomaly window) survives. + const md = renderReportMarkdown(stale); + expect(md).toContain("🔴 Flow unknown — data stale"); + expect(md).toContain("🔴 flow can't be assessed"); + expect(md).not.toContain("(last 40 min)"); // anomaly window gone + }); + + it("drops the CH-derived link from the VM when telemetry is stale", () => { + // flow's "concurrency" link is gone; only liveness' control-plane link may remain. + expect(stale.links.map((l) => l.key)).not.toContain("concurrency"); + }); + + it("flags the structured facts informational-only so an agent won't act on stale numbers", () => { + expect(stale.facts).toMatchObject({ trustworthy: false, staleReason: "telemetry_stale" }); + // fresh input is trustworthy. + expect(interpret(INPUT_A).facts).toMatchObject({ trustworthy: true }); + }); +}); + +describe("freshness unknown is distinct from lagging", () => { + it("renders 'data freshness unknown' in the summary, not 'data lagging'", () => { + const md = renderReportMarkdown(interpret({ ...INPUT_A, liveness: { telemetryAgeMs: null } })); + expect(md).toContain("data freshness unknown"); + expect(md).not.toContain("data lagging"); + }); + + it("does not change the flow severity policy (drainable crit still downgrades to warn)", () => { + const fresh = interpret(INPUT_A).findings.find((f) => f.type === "flow")!; + const unknown = interpret({ ...INPUT_A, liveness: { telemetryAgeMs: null } }).findings.find( + (f) => f.type === "flow" + )!; + expect(unknown.severity).toBe(fresh.severity); // warn, unaffected by unknown freshness + }); + + it("marks the liveness metric availability 'unknown' so value 0 isn't read as fresh", () => { + const unknown = interpret({ ...INPUT_A, liveness: { telemetryAgeMs: null } }); + const metric = unknown.metrics.find((m) => m.id === "liveness")!; + expect(metric.availability).toBe("unknown"); + expect(interpret(INPUT_A).metrics.find((m) => m.id === "liveness")!.availability).toBe( + "measured" + ); + }); +}); + +describe("zero baseline is not a false green (absolute floors)", () => { + it("pending spiking from a 0 baseline is not healthy", () => { + const vm = interpret({ + ...INPUT_B, + pending: { now: 6000, normal: 0, series: [0, 100, 6000], estimated: false }, + }); + expect(vm.findings.find((f) => f.type === "flow")!.severity).not.toBe("ok"); + }); + + it("failures spiking from a 0% baseline is not healthy", () => { + const vm = interpret({ ...INPUT_B, failures: { rate: 0.1, normalRate: 0, series: [0.1] } }); + expect(vm.findings.find((f) => f.type === "execution")!.severity).not.toBe("ok"); + }); +}); diff --git a/apps/webapp/test/reportHealthData.test.ts b/apps/webapp/test/reportHealthData.test.ts new file mode 100644 index 00000000000..ae29f7a3700 --- /dev/null +++ b/apps/webapp/test/reportHealthData.test.ts @@ -0,0 +1,242 @@ +import { describe, expect, it } from "vitest"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { + type HealthDeps, + type HealthQueryRunner, + loadHealthInput, +} from "~/presenters/v3/reports/health/health-data"; + +/** + * Exercises `loadHealthInput`'s ORCHESTRATION through its query seam (`HealthDeps`): + * source selection, snapshot fallback (empty + throw), dlq parsing, window-from-timeRange. + * The runner is injected at the IO boundary — SQL/TRQL translation and CH aggregation are + * tested by the query service and the `@internal/clickhouse` MV tests. + */ + +const NOW = new Date("2026-07-22T12:00:00.000Z"); + +const fakeEnv = { + id: "env_1", + slug: "prod", + organization: { id: "org_1" }, + project: { id: "proj_1" }, +} as unknown as AuthenticatedEnvironment; + +type Rows = Record<string, unknown>[]; + +/** Canned rows per query kind; the resolved timeRange is keyed off the period (7d = baseline). */ +function makeDeps(opts: { + runs: Rows; + runsSeries?: Rows; + envSeries?: Rows; + envScalar?: Rows; + dlqTotal?: Rows; + worst?: Rows; + liveWindowMin?: number; + pendingNow?: number; + throwOnEnv?: boolean; + redisThrows?: boolean; +}): HealthDeps { + const rangeFor = (period: string) => { + const min = period === "7d" ? 7 * 1440 : (opts.liveWindowMin ?? 60); + return { from: new Date(NOW.getTime() - min * 60_000), to: NOW }; + }; + const runQuery: HealthQueryRunner = async (_env, query, period) => { + const timeRange = rangeFor(period); + const wrap = (rows: Rows = []) => ({ rows, timeRange }); + const isEnv = query.includes("FROM env_metrics"); + if (isEnv && opts.throwOnEnv) throw new Error("env_metrics unavailable"); + if (query.includes("dlq_total")) return wrap(opts.dlqTotal); + if (isEnv && query.includes("timeBucket")) return wrap(opts.envSeries); + if (isEnv) return wrap(opts.envScalar ?? [{}]); + if (query.includes("FROM queue_metrics")) return wrap(opts.worst); + if (query.includes("task_identifier")) return wrap([]); + if (query.includes("FROM runs") && query.includes("timeBucket")) return wrap(opts.runsSeries); + return wrap(opts.runs); // runs scalar (live + baseline) + }; + const lengthOfEnvQueue = opts.redisThrows + ? async () => { + throw new Error("redis unavailable"); + } + : async () => opts.pendingNow ?? 0; + return { runQuery, lengthOfEnvQueue }; +} + +const RUNS_SCALAR: Rows = [ + { + start_latency_p95: 7000, + dur_p95: 1000, + failures: 1, + completed: 100, + triggered: 120, + last_activity: "2026-07-22 11:59:58", + }, +]; + +describe("loadHealthInput — orchestration (query seam)", () => { + it("measured path: queue_metrics source, real pending, parsed dlq, window from timeRange", async () => { + const input = await loadHealthInput( + fakeEnv, + "1h", + NOW, + makeDeps({ + runs: RUNS_SCALAR, + envSeries: [ + { t: "a", queued: 100, running: 50, throttled: 0, wait_p95: 5000 }, + { t: "b", queued: 300, running: 60, throttled: 1, wait_p95: 9000 }, + ], + envScalar: [{ wait_p95: 9000, avg_queued: 200, env_limit: 100 }], + dlqTotal: [{ dlq_total: 0 }], + worst: [ + { name: "email-sends", latest_queued: 82 }, + { name: "other", latest_queued: 18 }, + ], + pendingNow: 1910, + liveWindowMin: 60, + }) + ); + + expect(input.flowSource).toBe("queue_metrics_v1"); + expect(input.pending.estimated).toBe(false); + expect(input.pending.now).toBe(1910); + expect(input.flowEvidence.dlqDelta).toBe(0); + expect(input.flowEvidence.worstQueue).toEqual({ name: "email-sends", share: 0.82 }); + expect(input.windowMinutes).toBe(60); + expect(input.throughput.triggeredPerMin).toBeCloseTo(120 / 60); + }); + + it("dlq best-effort query returns nothing -> dlqDelta is null (unmeasured, no false 'nothing dead-lettered')", async () => { + const input = await loadHealthInput( + fakeEnv, + "1h", + NOW, + makeDeps({ + runs: RUNS_SCALAR, + envSeries: [{ t: "a", queued: 10, running: 5, throttled: 0, wait_p95: 100 }], + envScalar: [{ wait_p95: 100, avg_queued: 8, env_limit: 100 }], + dlqTotal: [], + }) + ); + + expect(input.flowSource).toBe("queue_metrics_v1"); + expect(input.flowEvidence.dlqDelta).toBeNull(); + }); + + it("no measured env rows -> snapshot fallback (estimated backlog)", async () => { + const input = await loadHealthInput( + fakeEnv, + "1h", + NOW, + makeDeps({ + runs: RUNS_SCALAR, + runsSeries: [{ t: "a", triggered: 10, completed: 8, start_latency_p95: 3000, failures: 0 }], + envSeries: [], // pipeline hasn't populated env_metrics for this env yet + }) + ); + + expect(input.flowSource).toBe("snapshot+runs"); + expect(input.pending.estimated).toBe(true); + }); + + it("env_metrics query throws -> snapshot fallback, never a 500 (bug-2 guard)", async () => { + const input = await loadHealthInput( + fakeEnv, + "1h", + NOW, + makeDeps({ + runs: RUNS_SCALAR, + runsSeries: [{ t: "a", triggered: 10, completed: 8, start_latency_p95: 3000, failures: 0 }], + throwOnEnv: true, + }) + ); + + expect(input.flowSource).toBe("snapshot+runs"); + expect(input.pending.estimated).toBe(true); + }); + + it("windowMinutes comes from the resolved (clipped) timeRange, not the period string", async () => { + const input = await loadHealthInput( + fakeEnv, + "1h", + NOW, + makeDeps({ + runs: RUNS_SCALAR, + envSeries: [{ t: "a", queued: 10, running: 5, throttled: 0, wait_p95: 100 }], + envScalar: [{ wait_p95: 100, avg_queued: 8, env_limit: 100 }], + dlqTotal: [{ dlq_total: 3 }], + liveWindowMin: 45, + }) + ); + + expect(input.windowMinutes).toBe(45); + expect(input.flowEvidence.dlqDelta).toBe(3); + }); + + it("telemetry age is measured from the newest ROW, not the query window end (= NOW)", async () => { + const OLD = "2026-07-22 11:00:00"; // 60 min before NOW, though the query window ends at NOW + const input = await loadHealthInput( + fakeEnv, + "1h", + NOW, + makeDeps({ + runs: [{ ...RUNS_SCALAR[0], last_activity: OLD }], + envSeries: [{ t: "a", queued: 10, running: 5, throttled: 0, wait_p95: 100 }], + envScalar: [{ wait_p95: 100, avg_queued: 8, env_limit: 100, last_bucket: OLD }], + }) + ); + // ~60 min from the row timestamp, NOT ~0 from timeRange.to + expect(input.liveness.telemetryAgeMs).toBeGreaterThan(50 * 60_000); + }); + + it("terminal runs (failed/canceled) do not accumulate as snapshot backlog", async () => { + const input = await loadHealthInput( + fakeEnv, + "1h", + NOW, + makeDeps({ + runs: RUNS_SCALAR, + // every triggered run also FINISHED this bucket (some failed) -> proxy stays flat at 0 + runsSeries: [ + { + t: "a", + triggered: 50, + completed: 30, + finished: 50, + start_latency_p95: 1000, + failures: 20, + }, + { + t: "b", + triggered: 40, + completed: 25, + finished: 40, + start_latency_p95: 1000, + failures: 15, + }, + ], + envSeries: [], // force the snapshot path + }) + ); + expect(input.flowSource).toBe("snapshot+runs"); + expect(Math.max(...input.pending.series)).toBe(0); + }); + + it("Redis rejection falls back to the latest measured queued, not a confident 0", async () => { + const input = await loadHealthInput( + fakeEnv, + "1h", + NOW, + makeDeps({ + runs: RUNS_SCALAR, + envSeries: [ + { t: "a", queued: 10, running: 5, throttled: 0, wait_p95: 100 }, + { t: "b", queued: 900, running: 5, throttled: 0, wait_p95: 100 }, + ], + envScalar: [{ wait_p95: 100, avg_queued: 8, env_limit: 100 }], + redisThrows: true, + }) + ); + expect(input.flowSource).toBe("queue_metrics_v1"); + expect(input.pending.now).toBe(900); + }); +}); diff --git a/apps/webapp/test/useTableSort.test.ts b/apps/webapp/test/useTableSort.test.ts new file mode 100644 index 00000000000..4b6c83666d9 --- /dev/null +++ b/apps/webapp/test/useTableSort.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { compareColumn, sortRows, type SortColumn } from "~/components/primitives/useTableSort"; + +type Row = { id: number; name: string | null; queued: number | null }; + +const rows: Row[] = [ + { id: 0, name: "banana", queued: 3 }, + { id: 1, name: "Apple", queued: null }, + { id: 2, name: "cherry", queued: 3 }, + { id: 3, name: null, queued: 1 }, +]; + +describe("sortRows", () => { + it("sorts numbers ascending with nulls last", () => { + const column: SortColumn<Row> = { key: "queued", type: "number", value: (r) => r.queued }; + expect(sortRows(rows, column, "asc").map((r) => r.id)).toEqual([3, 0, 2, 1]); + }); + + it("keeps nulls last even when descending", () => { + const column: SortColumn<Row> = { key: "queued", type: "number", value: (r) => r.queued }; + // 3 and 0 both have queued=3; stable order (0 before 2) is preserved, null (id 1) stays last. + expect(sortRows(rows, column, "desc").map((r) => r.id)).toEqual([0, 2, 3, 1]); + }); + + it("sorts alphabetically case-insensitively with empty/null last", () => { + const column: SortColumn<Row> = { key: "name", type: "alpha", value: (r) => r.name }; + expect(sortRows(rows, column, "asc").map((r) => r.id)).toEqual([1, 0, 2, 3]); + expect(sortRows(rows, column, "desc").map((r) => r.id)).toEqual([2, 0, 1, 3]); + }); + + it("is stable for equal values (preserves original order)", () => { + const column: SortColumn<Row> = { key: "queued", type: "number", value: () => 5 }; + expect(sortRows(rows, column, "asc").map((r) => r.id)).toEqual([0, 1, 2, 3]); + }); + + it("supports a custom comparator", () => { + // Sort by name length. + const column: SortColumn<Row> = { + key: "name", + type: "custom", + compare: (a, b) => (a.name?.length ?? 0) - (b.name?.length ?? 0), + }; + expect(sortRows(rows, column, "asc").map((r) => r.id)).toEqual([3, 1, 0, 2]); + }); +}); + +describe("compareColumn", () => { + it("treats NaN as null (sorts last)", () => { + const column: SortColumn<Row> = { key: "queued", type: "number", value: (r) => r.queued }; + const withNaN: Row = { id: 9, name: "x", queued: NaN }; + const normal: Row = { id: 10, name: "y", queued: 1 }; + expect(compareColumn(column, withNaN, normal, "asc")).toBe(1); + expect(compareColumn(column, withNaN, normal, "desc")).toBe(1); + }); +}); diff --git a/docs/v3-openapi.yaml b/docs/v3-openapi.yaml index aeb7b3d979a..bb42f7af2ca 100644 --- a/docs/v3-openapi.yaml +++ b/docs/v3-openapi.yaml @@ -3001,7 +3001,10 @@ paths: type: integer minimum: 0 maximum: 100000 - description: The new concurrency limit to set for the queue + description: | + The new concurrency limit to set for the queue. It may not exceed your + environment's maximum concurrency limit: a higher value is rejected with a + 400, not capped to the maximum. responses: "200": description: Concurrency limit overridden successfully @@ -3010,7 +3013,9 @@ paths: schema: "$ref": "#/components/schemas/QueueObject" "400": - description: Invalid request parameters + description: | + Invalid request parameters, or the requested concurrency limit exceeds the + environment's maximum concurrency limit. "401": description: Unauthorized request "404": diff --git a/internal-packages/clickhouse/schema/036_create_queue_metrics_v1.sql b/internal-packages/clickhouse/schema/036_create_queue_metrics_v1.sql new file mode 100644 index 00000000000..8ea1e65f09f --- /dev/null +++ b/internal-packages/clickhouse/schema/036_create_queue_metrics_v1.sql @@ -0,0 +1,267 @@ +-- +goose Up + +-- Queue metrics: raw landing table -> MV -> aggregated read target (mirrors +-- llm_model_aggregates_v1, migration 027). Raw rows feed an MV on insert, and +-- reads hit the aggregated table. + +-- Short-TTL raw landing, one row per stream entry. non_replicated_deduplication_window +-- makes consumer replays idempotent via insert_deduplication_token. +CREATE TABLE IF NOT EXISTS trigger_dev.queue_metrics_raw_v1 +( + organization_id LowCardinality(String), + project_id LowCardinality(String), + environment_id String CODEC(ZSTD(1)), + queue_name String CODEC(ZSTD(1)), + concurrency_key String DEFAULT '' CODEC(ZSTD(1)), -- per-key attribution ('' = base/whole-queue row) + event_time DateTime CODEC(Delta(4), ZSTD(1)), + order_key UInt64 DEFAULT 0, -- stream-id composite (ms*1e6+seq), deltaSumTimestamp ordering key + op LowCardinality(String), -- gauge | enqueue | started | ack | nack | dlq + running UInt32 DEFAULT 0, + queued UInt32 DEFAULT 0, + queue_limit UInt32 DEFAULT 0, + env_running UInt32 DEFAULT 0, + env_queued UInt32 DEFAULT 0, + env_limit UInt32 DEFAULT 0, + throttled UInt8 DEFAULT 0, -- 1 on a gauge emission with running>=limit AND queued>0 + ck_backlogged UInt32 DEFAULT 0, -- gauge on CK queues: distinct concurrency keys with queued work + ck_max_wait_ms UInt32 DEFAULT 0, -- gauge on CK queues: most-starved key's head-of-line wait + wait_ms UInt32 DEFAULT 0, -- set on op='started' (scheduling delay) + cumulative UInt64 DEFAULT 0 -- monotonic per-(queue,op) odometer on a counter op, diffed at read time +) +ENGINE = MergeTree() +PARTITION BY toDate(event_time) +ORDER BY (organization_id, project_id, environment_id, queue_name, event_time) +TTL event_time + INTERVAL 6 HOUR +SETTINGS non_replicated_deduplication_window = 1000, ttl_only_drop_parts = 1; + +-- (2) Aggregated read target (TRQL/dashboards query this). +CREATE TABLE IF NOT EXISTS trigger_dev.queue_metrics_v1 +( + organization_id LowCardinality(String), + project_id LowCardinality(String), + environment_id String CODEC(ZSTD(1)), + queue_name String CODEC(ZSTD(1)), + bucket_start DateTime CODEC(Delta(4), ZSTD(1)), + + -- Cumulative-counter deltas: each op maintains a monotonic odometer, and deltaSumTimestamp + -- sums positive consecutive deltas (ignoring resets) ordered by event_time, so a lost + -- reading self-heals (the next surviving reading restates the total). Read with + -- deltaSumTimestampMerge(<col>), never sum(). + enqueue_delta AggregateFunction(deltaSumTimestamp, UInt64, UInt64), + started_delta AggregateFunction(deltaSumTimestamp, UInt64, UInt64), + ack_delta AggregateFunction(deltaSumTimestamp, UInt64, UInt64), + nack_delta AggregateFunction(deltaSumTimestamp, UInt64, UInt64), + dlq_delta AggregateFunction(deltaSumTimestamp, UInt64, UInt64), + throttled_count SimpleAggregateFunction(sum, UInt64), + + max_queued SimpleAggregateFunction(max, UInt32), + max_running SimpleAggregateFunction(max, UInt32), + max_limit SimpleAggregateFunction(max, UInt32), + max_env_queued SimpleAggregateFunction(max, UInt32), + max_env_running SimpleAggregateFunction(max, UInt32), + max_env_limit SimpleAggregateFunction(max, UInt32), + max_ck_backlogged SimpleAggregateFunction(max, UInt32), + max_ck_wait_ms SimpleAggregateFunction(max, UInt32), + + wait_ms_sum SimpleAggregateFunction(sum, UInt64), + wait_ms_count SimpleAggregateFunction(sum, UInt64), + wait_quantiles AggregateFunction(quantiles(0.5, 0.9, 0.95, 0.99), UInt32) +) +ENGINE = AggregatingMergeTree() +PARTITION BY toDate(bucket_start) +ORDER BY (organization_id, project_id, environment_id, queue_name, bucket_start) +TTL bucket_start + INTERVAL 30 DAY +SETTINGS ttl_only_drop_parts = 1, non_replicated_deduplication_window = 1000; + +-- (3) MV: raw -> aggregated, 10s buckets. +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_mv_v1 +TO trigger_dev.queue_metrics_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + +-- (4) Env-level 10s rollup (no queue dimension) for header tiles/saturation charts. +-- Row count is queue-independent (~8640/day/env), so full granularity stays cheap at any range. +-- No counter deltas on purpose: cross-queue deltaSumTimestamp state merges mix unrelated +-- odometers (env totals must GROUP BY queue then sum). TDigest because an env-level +-- reservoir absorbs every sample in the environment. +CREATE TABLE IF NOT EXISTS trigger_dev.env_metrics_v1 +( + organization_id LowCardinality(String), + project_id LowCardinality(String), + environment_id String CODEC(ZSTD(1)), + bucket_start DateTime CODEC(Delta(4), ZSTD(1)), + + max_env_queued SimpleAggregateFunction(max, UInt32), + max_env_running SimpleAggregateFunction(max, UInt32), + max_env_limit SimpleAggregateFunction(max, UInt32), + throttled_count SimpleAggregateFunction(sum, UInt64), + + wait_ms_sum SimpleAggregateFunction(sum, UInt64), + wait_ms_count SimpleAggregateFunction(sum, UInt64), + wait_quantiles AggregateFunction(quantilesTDigest(0.5, 0.9, 0.95, 0.99), UInt32) +) +ENGINE = AggregatingMergeTree() +PARTITION BY toDate(bucket_start) +ORDER BY (organization_id, project_id, environment_id, bucket_start) +TTL bucket_start + INTERVAL 30 DAY +SETTINGS ttl_only_drop_parts = 1, non_replicated_deduplication_window = 1000; + +-- (5) MV: raw -> env rollup. +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.env_metrics_mv_v1 +TO trigger_dev.env_metrics_v1 AS +SELECT + organization_id, project_id, environment_id, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + sum(throttled) AS throttled_count, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesTDigestStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, bucket_start; + +-- (6) Per-queue 5m rollup, exact column mirror of queue_metrics_v1, for ranking and +-- env-wide GROUP BY queue reads at long ranges. +CREATE TABLE IF NOT EXISTS trigger_dev.queue_metrics_5m_v1 +( + organization_id LowCardinality(String), + project_id LowCardinality(String), + environment_id String CODEC(ZSTD(1)), + queue_name String CODEC(ZSTD(1)), + bucket_start DateTime CODEC(Delta(4), ZSTD(1)), + + enqueue_delta AggregateFunction(deltaSumTimestamp, UInt64, UInt64), + started_delta AggregateFunction(deltaSumTimestamp, UInt64, UInt64), + ack_delta AggregateFunction(deltaSumTimestamp, UInt64, UInt64), + nack_delta AggregateFunction(deltaSumTimestamp, UInt64, UInt64), + dlq_delta AggregateFunction(deltaSumTimestamp, UInt64, UInt64), + throttled_count SimpleAggregateFunction(sum, UInt64), + + max_queued SimpleAggregateFunction(max, UInt32), + max_running SimpleAggregateFunction(max, UInt32), + max_limit SimpleAggregateFunction(max, UInt32), + max_env_queued SimpleAggregateFunction(max, UInt32), + max_env_running SimpleAggregateFunction(max, UInt32), + max_env_limit SimpleAggregateFunction(max, UInt32), + max_ck_backlogged SimpleAggregateFunction(max, UInt32), + max_ck_wait_ms SimpleAggregateFunction(max, UInt32), + + wait_ms_sum SimpleAggregateFunction(sum, UInt64), + wait_ms_count SimpleAggregateFunction(sum, UInt64), + wait_quantiles AggregateFunction(quantiles(0.5, 0.9, 0.95, 0.99), UInt32) +) +ENGINE = AggregatingMergeTree() +PARTITION BY toDate(bucket_start) +ORDER BY (organization_id, project_id, environment_id, queue_name, bucket_start) +TTL bucket_start + INTERVAL 30 DAY +SETTINGS ttl_only_drop_parts = 1, non_replicated_deduplication_window = 1000; + +-- (7) MV: raw -> 5m rollup. MUST read raw, never cascade off queue_metrics_v1 with +-- -MergeState: MV GROUP BY merges states in hash order, and out-of-time-order +-- deltaSumTimestamp merges double-count bridging spans (verified 3x inflation). +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_5m_mv_v1 +TO trigger_dev.queue_metrics_5m_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + + +-- (8) Per-concurrency-key 10s tier. Rows are activity-bound (a (queue, key, bucket) row +-- exists only when that key had an event in that bucket), so user-controlled key +-- cardinality cannot inflate it beyond event volume (~19 bytes/event measured). +-- Lean columns: no nack/dlq deltas and no per-key quantile states (mean wait via sums). +CREATE TABLE IF NOT EXISTS trigger_dev.queue_metrics_ck_v1 +( + organization_id LowCardinality(String), + project_id LowCardinality(String), + environment_id String CODEC(ZSTD(1)), + queue_name String CODEC(ZSTD(1)), + concurrency_key String CODEC(ZSTD(1)), + bucket_start DateTime CODEC(Delta(4), ZSTD(1)), + + enqueue_delta AggregateFunction(deltaSumTimestamp, UInt64, UInt64), + started_delta AggregateFunction(deltaSumTimestamp, UInt64, UInt64), + ack_delta AggregateFunction(deltaSumTimestamp, UInt64, UInt64), + + max_queued SimpleAggregateFunction(max, UInt32), + max_running SimpleAggregateFunction(max, UInt32), + + wait_ms_sum SimpleAggregateFunction(sum, UInt64), + wait_ms_count SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree() +PARTITION BY toDate(bucket_start) +ORDER BY (organization_id, project_id, environment_id, queue_name, concurrency_key, bucket_start) +TTL bucket_start + INTERVAL 30 DAY +SETTINGS ttl_only_drop_parts = 1, non_replicated_deduplication_window = 1000; + +-- (9) MV: raw -> per-key tier. Only rows with a real key: per-key counter rows carry +-- per-key odometers (safe to merge within their own (queue, key) group), and per-key +-- gauge rows carry per-subqueue depth/running. +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_ck_mv_v1 +TO trigger_dev.queue_metrics_ck_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, concurrency_key, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack') AS ack_delta, + maxIf(queued, op = 'gauge') AS max_queued, + maxIf(running, op = 'gauge') AS max_running, + sumIf(wait_ms, op = 'started') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0) AS wait_ms_count +FROM trigger_dev.queue_metrics_raw_v1 +WHERE concurrency_key != '' +GROUP BY organization_id, project_id, environment_id, queue_name, concurrency_key, bucket_start; + +-- +goose Down +DROP VIEW IF EXISTS trigger_dev.queue_metrics_ck_mv_v1; +DROP TABLE IF EXISTS trigger_dev.queue_metrics_ck_v1; +DROP VIEW IF EXISTS trigger_dev.queue_metrics_5m_mv_v1; +DROP TABLE IF EXISTS trigger_dev.queue_metrics_5m_v1; +DROP VIEW IF EXISTS trigger_dev.env_metrics_mv_v1; +DROP TABLE IF EXISTS trigger_dev.env_metrics_v1; +DROP VIEW IF EXISTS trigger_dev.queue_metrics_mv_v1; +DROP TABLE IF EXISTS trigger_dev.queue_metrics_v1; +DROP TABLE IF EXISTS trigger_dev.queue_metrics_raw_v1; diff --git a/internal-packages/clickhouse/src/client/tsql.ts b/internal-packages/clickhouse/src/client/tsql.ts index c712820812f..ddf1d059b97 100644 --- a/internal-packages/clickhouse/src/client/tsql.ts +++ b/internal-packages/clickhouse/src/client/tsql.ts @@ -108,6 +108,11 @@ export interface ExecuteTSQLOptions<TOut extends z.ZodSchema> { * based on the span of the time range. */ timeRange?: TimeRange; + /** + * Opt-in: emit rows for empty time buckets in a top-level time-bucketed query + * (counters zero-fill, gauges carry forward). Off by default. + */ + fillGaps?: boolean; } /** @@ -192,6 +197,7 @@ export async function executeTSQL<TOut extends z.ZodSchema>( fieldMappings: options.fieldMappings, whereClauseFallback: options.whereClauseFallback, timeRange: options.timeRange, + fillGaps: options.fillGaps, }); generatedSql = sql; diff --git a/internal-packages/clickhouse/src/index.ts b/internal-packages/clickhouse/src/index.ts index bc8b6e662c8..49d2d7c02be 100644 --- a/internal-packages/clickhouse/src/index.ts +++ b/internal-packages/clickhouse/src/index.ts @@ -33,6 +33,15 @@ import { } from "./taskEvents.js"; import { insertMetrics } from "./metrics.js"; import { insertLlmMetrics } from "./llmMetrics.js"; +import { + insertQueueMetricsRaw, + getQueueListMetricsSummary, + getQueueDepthSparklines, + getQueueRanking, + getQueueRankingNames, + getQueueRankingCount, + getConcurrencyKeyRanking, +} from "./queueMetrics.js"; import { getSessionTagsQueryBuilder, getSessionsCountQueryBuilder, @@ -66,6 +75,7 @@ export type * from "./taskRuns.js"; export type * from "./taskEvents.js"; export type * from "./metrics.js"; export type * from "./llmMetrics.js"; +export type * from "./queueMetrics.js"; export type * from "./llmModelAggregates.js"; export type * from "./errors.js"; export type * from "./sessions.js"; @@ -262,6 +272,18 @@ export class ClickHouse { }; } + get queueMetrics() { + return { + insertRaw: insertQueueMetricsRaw(this.writer), + listSummary: getQueueListMetricsSummary(this.reader), + depthSparklines: getQueueDepthSparklines(this.reader), + ranking: getQueueRanking(this.reader), + rankingNames: getQueueRankingNames(this.reader), + rankingCount: getQueueRankingCount(this.reader), + concurrencyKeyRanking: getConcurrencyKeyRanking(this.reader), + }; + } + get llmModelAggregates() { return { globalMetrics: getGlobalModelMetrics(this.reader), diff --git a/internal-packages/clickhouse/src/queueMetrics.test.ts b/internal-packages/clickhouse/src/queueMetrics.test.ts new file mode 100644 index 00000000000..00532041e44 --- /dev/null +++ b/internal-packages/clickhouse/src/queueMetrics.test.ts @@ -0,0 +1,525 @@ +import { clickhouseTest } from "@internal/testcontainers"; +import { z } from "zod"; +import { ClickHouse } from "./index.js"; +import type { QueueMetricsRawV1Input } from "./queueMetrics.js"; + +const ORG = "org_qm"; +const PROJECT = "project_qm"; +const ENV = "env_qm"; +const EVENT_TIME = "2026-06-30 12:00:05"; // all rows land in the 10s bucket starting 12:00:00 + +function base(op: QueueMetricsRawV1Input["op"], queue: string): QueueMetricsRawV1Input { + return { + organization_id: ORG, + project_id: PROJECT, + environment_id: ENV, + queue_name: queue, + event_time: EVENT_TIME, + op, + }; +} + +// Cumulative counters: each op keeps a monotonic per-(queue,op) odometer, so a counter row +// carries the running total in `cumulative`. deltaSumTimestamp reconstructs the increase +// (last - first) from a seeded cum=0 baseline; order_key orders readings within an op. +let orderKey = 0; +function counter( + op: QueueMetricsRawV1Input["op"], + queue: string, + total: number, + waits?: number[] +): QueueMetricsRawV1Input[] { + const rows: QueueMetricsRawV1Input[] = [ + { ...base(op, queue), cumulative: 0, order_key: orderKey++ }, + ]; + for (let cum = 1; cum <= total; cum++) { + rows.push({ + ...base(op, queue), + cumulative: cum, + order_key: orderKey++, + ...(waits ? { wait_ms: waits[cum - 1] } : {}), + }); + } + return rows; +} + +const aggregatedRow = z.object({ + enqueue_count: z.coerce.number(), + started_count: z.coerce.number(), + ack_count: z.coerce.number(), + nack_count: z.coerce.number(), + dlq_count: z.coerce.number(), + throttled_count: z.coerce.number(), + max_running: z.coerce.number(), + max_queued: z.coerce.number(), + max_limit: z.coerce.number(), + max_env_running: z.coerce.number(), + max_env_queued: z.coerce.number(), + max_env_limit: z.coerce.number(), + max_ck_backlogged: z.coerce.number(), + max_ck_wait_ms: z.coerce.number(), + wait_ms_sum: z.coerce.number(), + wait_ms_count: z.coerce.number(), + wait_p50: z.coerce.number(), + wait_p90: z.coerce.number(), + wait_p95: z.coerce.number(), + wait_p99: z.coerce.number(), +}); + +function readAggregated(ch: ClickHouse) { + return ch.reader.query({ + name: "read-queue-metrics-aggregated", + query: `SELECT + deltaSumTimestampMerge(enqueue_delta) AS enqueue_count, + deltaSumTimestampMerge(started_delta) AS started_count, + deltaSumTimestampMerge(ack_delta) AS ack_count, + deltaSumTimestampMerge(nack_delta) AS nack_count, + deltaSumTimestampMerge(dlq_delta) AS dlq_count, + sum(throttled_count) AS throttled_count, + max(max_running) AS max_running, + max(max_queued) AS max_queued, + max(max_limit) AS max_limit, + max(max_env_running) AS max_env_running, + max(max_env_queued) AS max_env_queued, + max(max_env_limit) AS max_env_limit, + max(max_ck_backlogged) AS max_ck_backlogged, + max(max_ck_wait_ms) AS max_ck_wait_ms, + sum(wait_ms_sum) AS wait_ms_sum, + sum(wait_ms_count) AS wait_ms_count, + quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles) AS wait_arr, + wait_arr[1] AS wait_p50, + wait_arr[2] AS wait_p90, + wait_arr[3] AS wait_p95, + wait_arr[4] AS wait_p99 + FROM trigger_dev.queue_metrics_v1 + WHERE queue_name = {queueName: String} + GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start`, + schema: aggregatedRow, + params: z.object({ queueName: z.string() }), + }); +} + +// Synchronous insert so the MV-populated rows are queryable immediately. +const SYNC = { params: { clickhouse_settings: { async_insert: 0 as const } } }; + +describe("queue_metrics_v1", () => { + clickhouseTest( + "buckets counters, gauges and wait percentiles via the MV", + async ({ clickhouseContainer }) => { + const ch = new ClickHouse({ url: clickhouseContainer.getConnectionUrl(), name: "test" }); + const queue = "queue-a"; + + const rows: QueueMetricsRawV1Input[] = [ + ...counter("enqueue", queue, 3), + ...counter("started", queue, 10, [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000]), + ...counter("ack", queue, 2), + ...counter("nack", queue, 1), + ...counter("dlq", queue, 1), + { + ...base("gauge", queue), + running: 8, + queued: 4, + queue_limit: 10, + env_running: 40, + env_queued: 10, + env_limit: 50, + throttled: 0, + ck_backlogged: 3, + ck_max_wait_ms: 2500, + }, + { + ...base("gauge", queue), + running: 10, + queued: 6, + queue_limit: 10, + env_running: 50, + env_queued: 20, + env_limit: 50, + throttled: 1, // running >= limit AND queued > 0 + ck_backlogged: 2, + ck_max_wait_ms: 1500, + }, + ]; + + const [insertError] = await ch.queueMetrics.insertRaw(rows, SYNC); + expect(insertError).toBeNull(); + + const [queryError, result] = await readAggregated(ch)({ queueName: queue }); + expect(queryError).toBeNull(); + expect(result).toHaveLength(1); + const row = result![0]!; + + expect(row.enqueue_count).toBe(3); + expect(row.started_count).toBe(10); + expect(row.ack_count).toBe(2); + expect(row.nack_count).toBe(1); + expect(row.dlq_count).toBe(1); + expect(row.throttled_count).toBe(1); + + expect(row.max_running).toBe(10); + expect(row.max_queued).toBe(6); + expect(row.max_limit).toBe(10); + expect(row.max_env_running).toBe(50); + expect(row.max_env_queued).toBe(20); + expect(row.max_env_limit).toBe(50); + expect(row.max_ck_backlogged).toBe(3); + expect(row.max_ck_wait_ms).toBe(2500); + + expect(row.wait_ms_sum).toBe(5500); + expect(row.wait_ms_count).toBe(10); + + // Percentiles over [100..1000]: monotonic and within the value range. + expect(row.wait_p50).toBeGreaterThanOrEqual(400); + expect(row.wait_p50).toBeLessThanOrEqual(650); + expect(row.wait_p90).toBeGreaterThanOrEqual(row.wait_p50); + expect(row.wait_p95).toBeGreaterThanOrEqual(row.wait_p90); + expect(row.wait_p99).toBeGreaterThanOrEqual(row.wait_p95); + expect(row.wait_p99).toBeLessThanOrEqual(1000); + + await ch.close(); + } + ); + + clickhouseTest( + "merges wait-quantile state across separate insert blocks", + async ({ clickhouseContainer }) => { + const ch = new ClickHouse({ url: clickhouseContainer.getConnectionUrl(), name: "test" }); + const queue = "queue-b"; + + // Cumulative odometer continues across the two insert blocks (baseline 0, then 1..10); + // deltaSumTimestamp state and quantile state merge across the parts into one bucket. + const startedRow = (cum: number, wait_ms?: number): QueueMetricsRawV1Input => ({ + ...base("started", queue), + cumulative: cum, + order_key: orderKey++, + ...(wait_ms !== undefined ? { wait_ms } : {}), + }); + + const [e1] = await ch.queueMetrics.insertRaw( + [startedRow(0), ...[100, 200, 300, 400, 500].map((w, i) => startedRow(i + 1, w))], + SYNC + ); + expect(e1).toBeNull(); + const [e2] = await ch.queueMetrics.insertRaw( + [600, 700, 800, 900, 1000].map((w, i) => startedRow(i + 6, w)), + SYNC + ); + expect(e2).toBeNull(); + + const [queryError, result] = await readAggregated(ch)({ queueName: queue }); + expect(queryError).toBeNull(); + expect(result).toHaveLength(1); + const row = result![0]!; + + // Both blocks contribute to one bucket: counts and sums add, quantile state merges. + expect(row.started_count).toBe(10); + expect(row.wait_ms_sum).toBe(5500); + expect(row.wait_ms_count).toBe(10); + expect(row.wait_p50).toBeGreaterThanOrEqual(400); + expect(row.wait_p50).toBeLessThanOrEqual(650); + expect(row.wait_p99).toBeGreaterThanOrEqual(row.wait_p50); + expect(row.wait_p99).toBeLessThanOrEqual(1000); + + await ch.close(); + } + ); + + clickhouseTest( + "5m and env rollups agree with the 10s tier, and env buckets are 10s", + async ({ clickhouseContainer }) => { + const ch = new ClickHouse({ url: clickhouseContainer.getConnectionUrl(), name: "test" }); + + // Own org so the env-level read (no queue filter) stays isolated from other tests. + const rollOrg = "org_qm_roll"; + const rows: QueueMetricsRawV1Input[] = [ + ...counter("started", "roll-a", 7, [100, 150, 200, 250, 300, 350, 400]), + ...counter("started", "roll-b", 3, [500, 600, 700]), + { + ...base("gauge", "roll-a"), + running: 4, + queued: 9, + env_running: 30, + env_limit: 50, + ck_backlogged: 5, + ck_max_wait_ms: 9000, + }, + { ...base("gauge", "roll-b"), running: 2, queued: 1, env_running: 45, env_limit: 50 }, + { + ...base("gauge", "roll-a"), + event_time: "2026-06-30 12:00:15", + running: 1, + queued: 2, + env_running: 20, + env_limit: 50, + ck_backlogged: 2, + ck_max_wait_ms: 3000, + }, + ].map((row) => ({ ...row, organization_id: rollOrg })); + const [insertError] = await ch.queueMetrics.insertRaw(rows, SYNC); + expect(insertError).toBeNull(); + + const perQueue = (table: string) => + ch.reader.query({ + name: "per-queue-both-tiers", + query: `SELECT queue_name, deltaSumTimestampMerge(started_delta) AS started + FROM ${table} + WHERE queue_name IN ('roll-a', 'roll-b') + GROUP BY queue_name ORDER BY queue_name`, + schema: z.object({ queue_name: z.string(), started: z.coerce.number() }), + })({}); + const [e10, rows10] = await perQueue("trigger_dev.queue_metrics_v1"); + const [e5m, rows5m] = await perQueue("trigger_dev.queue_metrics_5m_v1"); + expect(e10).toBeNull(); + expect(e5m).toBeNull(); + expect(rows10).toEqual([ + { queue_name: "roll-a", started: 7 }, + { queue_name: "roll-b", started: 3 }, + ]); + expect(rows5m).toEqual(rows10); + + // CK-health gauges roll into the 5m mirror too. + const [ckError, ckRows] = await ch.reader.query({ + name: "ck-5m-read", + query: `SELECT max(max_ck_backlogged) AS ck_keys, max(max_ck_wait_ms) AS ck_wait + FROM trigger_dev.queue_metrics_5m_v1 + WHERE queue_name = 'roll-a'`, + schema: z.object({ ck_keys: z.coerce.number(), ck_wait: z.coerce.number() }), + })({}); + expect(ckError).toBeNull(); + expect(ckRows![0]).toEqual({ ck_keys: 5, ck_wait: 9000 }); + + // Env-wide totals: sum of per-queue merges (a single merge across queues would mix + // odometers and double-count). + const [envTotalError, envTotal] = await ch.reader.query({ + name: "env-total-per-queue-sum", + query: `SELECT sum(started) AS started FROM ( + SELECT queue_name, deltaSumTimestampMerge(started_delta) AS started + FROM trigger_dev.queue_metrics_5m_v1 + WHERE queue_name IN ('roll-a', 'roll-b') + GROUP BY queue_name + )`, + schema: z.object({ started: z.coerce.number() }), + })({}); + expect(envTotalError).toBeNull(); + expect(envTotal![0]!.started).toBe(10); + + const [envError, envRows] = await ch.reader.query({ + name: "env-rollup-read", + query: `SELECT + max(max_env_running) AS max_env_running, + max(max_env_limit) AS max_env_limit, + uniqExact(bucket_start) AS buckets, + round(quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[4]) AS wait_p99 + FROM trigger_dev.env_metrics_v1 + WHERE organization_id = {org: String}`, + schema: z.object({ + max_env_running: z.coerce.number(), + max_env_limit: z.coerce.number(), + buckets: z.coerce.number(), + wait_p99: z.coerce.number(), + }), + params: z.object({ org: z.string() }), + })({ org: rollOrg }); + expect(envError).toBeNull(); + expect(envRows![0]!.max_env_running).toBe(45); + expect(envRows![0]!.max_env_limit).toBe(50); + // 12:00:05 and 12:00:15 land in separate 10s env buckets (12:00:00 and 12:00:10). + expect(envRows![0]!.buckets).toBe(2); + expect(envRows![0]!.wait_p99).toBeGreaterThanOrEqual(600); + expect(envRows![0]!.wait_p99).toBeLessThanOrEqual(1000); + + await ch.close(); + } + ); + + clickhouseTest( + "merged ranking returns the page and the windowed total in one query", + async ({ clickhouseContainer }) => { + const ch = new ClickHouse({ url: clickhouseContainer.getConnectionUrl(), name: "test" }); + + const gauge = (queue: string, queued: number, running: number): QueueMetricsRawV1Input => ({ + ...base("gauge", queue), + queued, + running, + }); + const [insertError] = await ch.queueMetrics.insertRaw( + [gauge("rank-low", 1, 0), gauge("rank-high", 50, 3), gauge("rank-mid", 10, 2)], + SYNC + ); + expect(insertError).toBeNull(); + + const args = { + organizationId: ORG, + projectId: PROJECT, + environmentId: ENV, + startTime: "2026-06-30 11:50:00", + nameContains: "rank-", + byQueuedOnly: 0, + }; + const [pageError, page] = await ch.queueMetrics.ranking({ ...args, limit: 2, offset: 0 }); + expect(pageError).toBeNull(); + expect(page).toEqual([ + { queue_name: "rank-high", ranked_total: 3 }, + { queue_name: "rank-mid", ranked_total: 3 }, + ]); + + const [countError, count] = await ch.queueMetrics.rankingCount(args); + expect(countError).toBeNull(); + expect(count![0]!.ranked).toBe(3); + + const [namesError, names] = await ch.queueMetrics.rankingNames({ ...args, limit: 10 }); + expect(namesError).toBeNull(); + expect(names!.map((r) => r.queue_name)).toEqual(["rank-high", "rank-mid", "rank-low"]); + + await ch.close(); + } + ); +}); + +describe("consumer retry idempotency", () => { + clickhouseTest( + "re-inserting a batch with the same dedup token does not inflate any tier", + async ({ clickhouseContainer }) => { + const ch = new ClickHouse({ url: clickhouseContainer.getConnectionUrl(), name: "test" }); + + const dedupOrg = "org_qm_dedup"; + const rows: QueueMetricsRawV1Input[] = [ + ...counter("started", "dedup-q", 3, [100, 200, 300]), + { ...base("gauge", "dedup-q"), running: 2, queued: 1, env_running: 5, env_limit: 10 }, + ].map((row) => ({ ...row, organization_id: dedupOrg })); + + const retrySettings = { + params: { + clickhouse_settings: { + async_insert: 0 as const, + insert_deduplication_token: "qm-test-retry-batch", + deduplicate_blocks_in_dependent_materialized_views: 1 as const, + }, + }, + }; + for (let attempt = 0; attempt < 3; attempt++) { + const [error] = await ch.queueMetrics.insertRaw(rows, retrySettings); + expect(error).toBeNull(); + } + + const [tiersError, tiers] = await ch.reader.query({ + name: "dedup-tier-counts", + query: `SELECT + (SELECT count() FROM trigger_dev.queue_metrics_v1 WHERE organization_id = {org: String}) AS rows_10s, + (SELECT count() FROM trigger_dev.queue_metrics_5m_v1 WHERE organization_id = {org: String}) AS rows_5m, + (SELECT count() FROM trigger_dev.env_metrics_v1 WHERE organization_id = {org: String}) AS rows_env, + (SELECT sum(wait_ms_count) FROM trigger_dev.env_metrics_v1 WHERE organization_id = {org: String}) AS wait_count, + (SELECT deltaSumTimestampMerge(started_delta) FROM trigger_dev.queue_metrics_v1 WHERE organization_id = {org: String}) AS started`, + schema: z.object({ + rows_10s: z.coerce.number(), + rows_5m: z.coerce.number(), + rows_env: z.coerce.number(), + wait_count: z.coerce.number(), + started: z.coerce.number(), + }), + params: z.object({ org: z.string() }), + })({ org: dedupOrg }); + expect(tiersError).toBeNull(); + const t = tiers![0]!; + // Without dedup windows on the MV targets, retries append copies: rows and sums triple. + expect(t.rows_10s).toBe(1); + expect(t.rows_5m).toBe(1); + expect(t.rows_env).toBe(1); + expect(t.wait_count).toBe(3); + expect(t.started).toBe(3); + + await ch.close(); + } + ); +}); + +describe("per-concurrency-key tier", () => { + clickhouseTest( + "per-key rows feed the ck tier without polluting per-queue counters or waits", + async ({ clickhouseContainer }) => { + const ch = new ClickHouse({ url: clickhouseContainer.getConnectionUrl(), name: "test" }); + const ckOrg = "org_qm_ck"; + const queue = "ck-tier-q"; + const withCk = (row: QueueMetricsRawV1Input, ck: string): QueueMetricsRawV1Input => ({ + ...row, + concurrency_key: ck, + }); + + // 5 started events on one queue across two keys (t1 x3, t2 x2). Each event lands as + // a base row (base odometer) + a per-key row (per-key odometer), both carrying wait, + // exactly like the consumer expansion. Baselines seed each odometer. + const rows: QueueMetricsRawV1Input[] = []; + let ok = 0; + const started = (cum: number, ck: string, ckcum: number, wait: number) => { + rows.push({ ...base("started", queue), cumulative: cum, order_key: ok, wait_ms: wait }); + rows.push( + withCk({ ...base("started", queue), cumulative: ckcum, order_key: ok, wait_ms: wait }, ck) + ); + ok++; + }; + rows.push({ ...base("started", queue), cumulative: 0, order_key: ok++ }); + rows.push(withCk({ ...base("started", queue), cumulative: 0, order_key: ok++ }, "t1")); + rows.push(withCk({ ...base("started", queue), cumulative: 0, order_key: ok++ }, "t2")); + started(1, "t1", 1, 100); + started(2, "t1", 2, 200); + started(3, "t2", 1, 300); + started(4, "t1", 3, 400); + started(5, "t2", 2, 500); + // Per-subqueue gauges carry the key. + rows.push(withCk({ ...base("gauge", queue), queued: 4, running: 1 }, "t1")); + rows.push(withCk({ ...base("gauge", queue), queued: 2, running: 0 }, "t2")); + + const [insertError] = await ch.queueMetrics.insertRaw( + rows.map((r) => ({ ...r, organization_id: ckOrg })), + SYNC + ); + expect(insertError).toBeNull(); + + const [perQueueError, perQueue] = await ch.reader.query({ + name: "ck-per-queue-read", + query: `SELECT + deltaSumTimestampMerge(started_delta) AS started, + sum(wait_ms_sum) AS wait_sum, + sum(wait_ms_count) AS wait_count, + max(max_queued) AS peak_queued + FROM trigger_dev.queue_metrics_v1 + WHERE organization_id = {org: String}`, + schema: z.object({ + started: z.coerce.number(), + wait_sum: z.coerce.number(), + wait_count: z.coerce.number(), + peak_queued: z.coerce.number(), + }), + params: z.object({ org: z.string() }), + })({ org: ckOrg }); + expect(perQueueError).toBeNull(); + // Base rows only: 5 events (not 10), waits counted once, per-key gauges still max in. + expect(perQueue![0]).toEqual({ started: 5, wait_sum: 1500, wait_count: 5, peak_queued: 4 }); + + const [ckError, ckRows] = await ch.reader.query({ + name: "ck-tier-read", + query: `SELECT concurrency_key, + deltaSumTimestampMerge(started_delta) AS started, + max(max_queued) AS peak_queued, + sum(wait_ms_sum) AS wait_sum + FROM trigger_dev.queue_metrics_ck_v1 + WHERE organization_id = {org: String} + GROUP BY concurrency_key ORDER BY concurrency_key`, + schema: z.object({ + concurrency_key: z.string(), + started: z.coerce.number(), + peak_queued: z.coerce.number(), + wait_sum: z.coerce.number(), + }), + params: z.object({ org: z.string() }), + })({ org: ckOrg }); + expect(ckError).toBeNull(); + expect(ckRows).toEqual([ + { concurrency_key: "t1", started: 3, peak_queued: 4, wait_sum: 700 }, + { concurrency_key: "t2", started: 2, peak_queued: 2, wait_sum: 800 }, + ]); + + await ch.close(); + } + ); +}); diff --git a/internal-packages/clickhouse/src/queueMetrics.ts b/internal-packages/clickhouse/src/queueMetrics.ts new file mode 100644 index 00000000000..39576b4a0a3 --- /dev/null +++ b/internal-packages/clickhouse/src/queueMetrics.ts @@ -0,0 +1,294 @@ +import { z } from "zod"; +import type { ClickhouseReader, ClickhouseWriter } from "./client/types.js"; + +export const QueueMetricsRawV1Input = z.object({ + organization_id: z.string(), + project_id: z.string(), + environment_id: z.string(), + queue_name: z.string(), + concurrency_key: z.string().optional(), + event_time: z.string(), + // Exact UInt64 ordering key; a string preserves precision past JS safe-integer range + // (see entryOrderKey). A plain number is still accepted for small test values. + order_key: z.union([z.string(), z.number()]).optional(), + op: z.enum(["gauge", "enqueue", "started", "ack", "nack", "dlq"]), + running: z.number().optional(), + queued: z.number().optional(), + queue_limit: z.number().optional(), + env_running: z.number().optional(), + env_queued: z.number().optional(), + env_limit: z.number().optional(), + throttled: z.number().optional(), + ck_backlogged: z.number().optional(), + ck_max_wait_ms: z.number().optional(), + wait_ms: z.number().optional(), + cumulative: z.number().optional(), +}); + +export type QueueMetricsRawV1Input = z.input<typeof QueueMetricsRawV1Input>; + +export function insertQueueMetricsRaw(ch: ClickhouseWriter) { + return ch.insertUnsafe<QueueMetricsRawV1Input>({ + name: "insertQueueMetricsRaw", + table: "trigger_dev.queue_metrics_raw_v1", + }); +} + +// --- Reads (Queues list metrics + health) --- + +const QueueMetricsListParams = z.object({ + organizationId: z.string(), + projectId: z.string(), + environmentId: z.string(), + queueNames: z.array(z.string()), + startTime: z.string(), + endTime: z.string(), +}); + +const QueueMetricsSummaryRow = z.object({ + queue_name: z.string(), + p50_wait_ms: z.coerce.number(), + p95_wait_ms: z.coerce.number(), + peak_queued: z.coerce.number(), + started_count: z.coerce.number(), + throttled_count: z.coerce.number(), +}); + +// Callers align window bounds to the bucket grid so repeated loads share cache entries. +const QUEUE_METRICS_CACHE_SETTINGS = { + use_query_cache: 1, + query_cache_ttl: 30, +} as const; + +/** Per-queue rollups over a window, for a fixed set of queues (the visible page). */ +export function getQueueListMetricsSummary(reader: ClickhouseReader) { + return reader.query({ + name: "getQueueListMetricsSummary", + query: `SELECT + queue_name, + round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[1]) AS p50_wait_ms, + round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95_wait_ms, + max(max_queued) AS peak_queued, + deltaSumTimestampMerge(started_delta) AS started_count, + sum(throttled_count) AS throttled_count + FROM trigger_dev.queue_metrics_v1 + WHERE organization_id = {organizationId: String} + AND project_id = {projectId: String} + AND environment_id = {environmentId: String} + AND queue_name IN {queueNames: Array(String)} + AND bucket_start >= {startTime: DateTime} + AND bucket_start < {endTime: DateTime} + GROUP BY queue_name`, + params: QueueMetricsListParams, + schema: QueueMetricsSummaryRow, + settings: QUEUE_METRICS_CACHE_SETTINGS, + }); +} + +const QueueDepthSparklineParams = QueueMetricsListParams.extend({ + bucketSeconds: z.number(), +}); + +const QueueDepthSparklineRow = z.object({ + queue_name: z.string(), + bucket: z.string(), + depth: z.coerce.number(), + throttled: z.coerce.number(), +}); + +/** + * Per-queue, per-bucket peak depth (carry-forward filled by the caller) plus the throttled + * count in each bucket, so the sparkline can tint the exact buckets where throttling occurred. + * The extra aggregate rides the same scan as the depth series — no additional round trip. + */ +export function getQueueDepthSparklines(reader: ClickhouseReader) { + return reader.query({ + name: "getQueueDepthSparklines", + query: `SELECT + queue_name, + toStartOfInterval(bucket_start, toIntervalSecond({bucketSeconds: UInt32})) AS bucket, + max(max_queued) AS depth, + sum(throttled_count) AS throttled + FROM trigger_dev.queue_metrics_v1 + WHERE organization_id = {organizationId: String} + AND project_id = {projectId: String} + AND environment_id = {environmentId: String} + AND queue_name IN {queueNames: Array(String)} + AND bucket_start >= {startTime: DateTime} + AND bucket_start < {endTime: DateTime} + GROUP BY queue_name, bucket + ORDER BY bucket`, + params: QueueDepthSparklineParams, + schema: QueueDepthSparklineRow, + settings: QUEUE_METRICS_CACHE_SETTINGS, + }); +} + +const QueueRankingParams = z.object({ + organizationId: z.string(), + projectId: z.string(), + environmentId: z.string(), + startTime: z.string(), + /** 1 = rank by peak backlog only; 0 = backlog + running ("busiest"). */ + byQueuedOnly: z.number(), + nameContains: z.string(), + limit: z.number(), + offset: z.number(), +}); + +const QueueRankingRow = z.object({ + queue_name: z.string(), + ranked_total: z.coerce.number(), +}); + +// Ranking reads the 5m rollup: a 15-minute window there costs ~30x fewer rows than the +// 10s table. +const RANKING_WHERE = `organization_id = {organizationId: String} + AND project_id = {projectId: String} + AND environment_id = {environmentId: String} + AND bucket_start >= {startTime: DateTime} + AND queue_name != '__overflow__' + AND ({nameContains: String} = '' OR positionCaseInsensitive(queue_name, {nameContains: String}) > 0)`; + +/** + * One page of queue names ranked by recent activity, with the total ranked count on + * every row (window function), so page + count cost a single scan. + */ +export function getQueueRanking(reader: ClickhouseReader) { + return reader.query({ + name: "getQueueRanking", + query: `SELECT queue_name, count() OVER () AS ranked_total + FROM ( + SELECT queue_name + FROM trigger_dev.queue_metrics_5m_v1 + WHERE ${RANKING_WHERE} + GROUP BY queue_name + ORDER BY + if({byQueuedOnly: UInt8} = 1, max(max_queued), max(max_queued) + max(max_running)) DESC, + queue_name ASC + ) + LIMIT {limit: UInt32} OFFSET {offset: UInt32}`, + params: QueueRankingParams, + schema: QueueRankingRow, + settings: QUEUE_METRICS_CACHE_SETTINGS, + }); +} + +const QueueRankingNamesParams = QueueRankingParams.omit({ byQueuedOnly: true, offset: true }); + +const QueueRankingNameRow = z.object({ + queue_name: z.string(), +}); + +/** All ranked queue names (activity order), used to exclude them from the alphabetical tail. */ +export function getQueueRankingNames(reader: ClickhouseReader) { + return reader.query({ + name: "getQueueRankingNames", + query: `SELECT queue_name + FROM trigger_dev.queue_metrics_5m_v1 + WHERE ${RANKING_WHERE} + GROUP BY queue_name + ORDER BY max(max_queued) + max(max_running) DESC, queue_name ASC + LIMIT {limit: UInt32}`, + params: QueueRankingNamesParams, + schema: QueueRankingNameRow, + settings: QUEUE_METRICS_CACHE_SETTINGS, + }); +} + +const QueueRankingCountParams = QueueRankingParams.omit({ + byQueuedOnly: true, + limit: true, + offset: true, +}); + +const QueueRankingCountRow = z.object({ + ranked: z.coerce.number(), +}); + +/** Ranked-queue count alone, for pages past the ranked head (approximate uniq is fine). */ +export function getQueueRankingCount(reader: ClickhouseReader) { + return reader.query({ + name: "getQueueRankingCount", + query: `SELECT uniq(queue_name) AS ranked + FROM trigger_dev.queue_metrics_5m_v1 + WHERE ${RANKING_WHERE}`, + params: QueueRankingCountParams, + schema: QueueRankingCountRow, + settings: QUEUE_METRICS_CACHE_SETTINGS, + }); +} + +// --- Per-concurrency-key ranking (the queue detail "Concurrency keys" table) --- + +const ConcurrencyKeyRankingParams = z.object({ + organizationId: z.string(), + projectId: z.string(), + environmentId: z.string(), + queueName: z.string(), + startTime: z.string(), + endTime: z.string(), + /** Case-insensitive substring filter on the key ('' = no filter). */ + nameContains: z.string(), + limit: z.number(), + offset: z.number(), +}); + +const ConcurrencyKeyRankingRow = z.object({ + concurrency_key: z.string(), + started: z.coerce.number(), + peak_backlog: z.coerce.number(), + peak_running: z.coerce.number(), + mean_wait_ms: z.coerce.number(), + ranked_total: z.coerce.number(), +}); + +// The per-key table (queue_metrics_ck_v1) is activity-bound and its ORDER BY starts with the +// tenant + queue, so filtering to one queue prunes to a contiguous index range — the aggregate +// is bounded by real activity, never by total key cardinality. There is no per-key 5m rollup, +// so this reads the 10s tier directly (the pre-existing LIMIT-50 query did the same). +const CK_RANKING_WHERE = `organization_id = {organizationId: String} + AND project_id = {projectId: String} + AND environment_id = {environmentId: String} + AND queue_name = {queueName: String} + AND bucket_start >= {startTime: DateTime} + AND bucket_start < {endTime: DateTime} + AND ({nameContains: String} = '' OR positionCaseInsensitive(concurrency_key, {nameContains: String}) > 0)`; + +/** + * One page of a queue's concurrency keys ranked by peak backlog over the window, with the total + * ranked-key count on every row (window function) so page + count cost a single scan — the same + * shape as getQueueRanking. The `concurrency_key ASC` tiebreak makes OFFSET paging stable across + * keys that share a peak. Range stats (started/peak_backlog/peak_running/mean wait) come back on + * the same rows; live "now" counts are enriched per page from Redis by the caller. + */ +export function getConcurrencyKeyRanking(reader: ClickhouseReader) { + return reader.query({ + name: "getConcurrencyKeyRanking", + query: `SELECT + concurrency_key, + started, + peak_backlog, + peak_running, + mean_wait_ms, + count() OVER () AS ranked_total + FROM ( + SELECT + concurrency_key, + deltaSumTimestampMerge(started_delta) AS started, + max(max_queued) AS peak_backlog, + max(max_running) AS peak_running, + if(sum(wait_ms_count) > 0, round(sum(wait_ms_sum) / sum(wait_ms_count)), 0) AS mean_wait_ms + FROM trigger_dev.queue_metrics_ck_v1 + WHERE ${CK_RANKING_WHERE} + GROUP BY concurrency_key + ORDER BY peak_backlog DESC, concurrency_key ASC + ) + LIMIT {limit: UInt32} OFFSET {offset: UInt32}`, + params: ConcurrencyKeyRankingParams, + schema: ConcurrencyKeyRankingRow, + settings: QUEUE_METRICS_CACHE_SETTINGS, + }); +} + +// (per-queue detail series is now fetched via TRQL + fillGaps from the metric resource route) diff --git a/internal-packages/database/prisma/migrations/20260716120000_add_task_queue_concurrency_limit_override_percent/migration.sql b/internal-packages/database/prisma/migrations/20260716120000_add_task_queue_concurrency_limit_override_percent/migration.sql new file mode 100644 index 00000000000..cf1aa0ed84e --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260716120000_add_task_queue_concurrency_limit_override_percent/migration.sql @@ -0,0 +1,5 @@ +-- Adds a nullable column to store a queue concurrency-limit override expressed as a percentage +-- of the environment limit. This percentage is the source of truth; the absolute +-- "concurrencyLimit" is materialized from it at save time. Additive and nullable, so existing +-- absolute overrides are unaffected. Decimal(5,2) supports fractional percentages (0.01–100.00). +ALTER TABLE "TaskQueue" ADD COLUMN IF NOT EXISTS "concurrencyLimitOverridePercent" DECIMAL(5,2); diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 7d6f4ac5493..2998df20937 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -1831,14 +1831,18 @@ model TaskQueue { runtimeEnvironmentId String /// Represents the current concurrency limit for the queue - concurrencyLimit Int? + concurrencyLimit Int? /// When the concurrency limit was overridden - concurrencyLimitOverriddenAt DateTime? + concurrencyLimitOverriddenAt DateTime? /// Who overrode the concurrency limit (will be null if overridden via the API) - concurrencyLimitOverriddenBy String? + concurrencyLimitOverriddenBy String? /// If concurrencyLimit is overridden, this is the overridden value - concurrencyLimitBase Int? - rateLimit Json? + concurrencyLimitBase Int? + /// If the override was expressed as a percentage of the environment limit, this stores that + /// percentage (the source of truth). The absolute concurrencyLimit is materialized from it. + /// Decimal(5,2) allows fractional percentages like 12.50% (0.01–100.00). + concurrencyLimitOverridePercent Decimal? @db.Decimal(5, 2) + rateLimit Json? paused Boolean @default(false) diff --git a/internal-packages/metrics-pipeline/package.json b/internal-packages/metrics-pipeline/package.json new file mode 100644 index 00000000000..10a7c137a1f --- /dev/null +++ b/internal-packages/metrics-pipeline/package.json @@ -0,0 +1,33 @@ +{ + "name": "@internal/metrics-pipeline", + "private": true, + "version": "0.0.1", + "main": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", + "type": "module", + "exports": { + ".": { + "@triggerdotdev/source": "./src/index.ts", + "import": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", + "default": "./dist/src/index.js" + } + }, + "dependencies": { + "@internal/redis": "workspace:*", + "@internal/tracing": "workspace:*", + "@trigger.dev/core": "workspace:*" + }, + "devDependencies": { + "@internal/testcontainers": "workspace:*", + "rimraf": "6.0.1" + }, + "scripts": { + "clean": "rimraf dist", + "typecheck": "tsc --noEmit -p tsconfig.build.json", + "test": "vitest --sequence.concurrent=false --no-file-parallelism", + "test:coverage": "vitest --sequence.concurrent=false --no-file-parallelism --coverage.enabled", + "build": "pnpm run clean && tsc -p tsconfig.build.json", + "dev": "tsc --watch -p tsconfig.build.json" + } +} diff --git a/internal-packages/metrics-pipeline/src/cachedValue.ts b/internal-packages/metrics-pipeline/src/cachedValue.ts new file mode 100644 index 00000000000..7f7bbb07903 --- /dev/null +++ b/internal-packages/metrics-pipeline/src/cachedValue.ts @@ -0,0 +1,125 @@ +import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis"; +import { Logger } from "@trigger.dev/core/logger"; + +export type CachedRedisValueOptions<T> = { + redis: RedisOptions; + key: string; + parse: (raw: string | null) => T; + defaultValue: T; + cacheTtlMs?: number; + logger?: Logger; + loggerName?: string; +}; + +// Reads a Redis key with a short stale-while-revalidate cache and a synchronous getter for +// hot paths. Warms eagerly on construction; concurrent refreshes dedupe onto one GET so an +// awaited refresh always resolves to a completed read. +export class CachedRedisValue<T> { + private readonly redis: Redis; + private readonly key: string; + private readonly parse: (raw: string | null) => T; + private readonly cacheTtlMs: number; + private readonly logger: Logger; + private value: T; + private lastFetchedAt = 0; + private refreshPromise?: Promise<T>; + + constructor(options: CachedRedisValueOptions<T>) { + this.logger = options.logger ?? new Logger(options.loggerName ?? "CachedRedisValue", "warn"); + this.redis = createRedisClient( + { ...options.redis, keyPrefix: undefined }, + { + onError: (error) => + this.logger.error("cached value redis error", { error, key: options.key }), + } + ); + this.key = options.key; + this.parse = options.parse; + this.cacheTtlMs = options.cacheTtlMs ?? 10_000; + this.value = options.defaultValue; + void this.refresh(); + } + + get(): T { + if (Date.now() - this.lastFetchedAt > this.cacheTtlMs) { + void this.refresh(); + } + return this.value; + } + + async refresh(): Promise<T> { + if (this.refreshPromise) return this.refreshPromise; + this.refreshPromise = this.#doRefresh(); + try { + return await this.refreshPromise; + } finally { + this.refreshPromise = undefined; + } + } + + async #doRefresh(): Promise<T> { + try { + this.value = this.parse(await this.redis.get(this.key)); + } catch (error) { + this.logger.debug("cached value refresh failed, keeping cached value", { + error, + key: this.key, + }); + } finally { + this.lastFetchedAt = Date.now(); + } + return this.value; + } + + async close(): Promise<void> { + await this.redis.quit(); + } +} + +export type CachedRedisNumberOptions = { + redis: RedisOptions; + key: string; + defaultValue: number; + min?: number; + max?: number; + cacheTtlMs?: number; + logger?: Logger; +}; + +// Live-tunable numeric value, clamped to [min,max]; falls back to defaultValue on a +// missing/unparseable key. Exposes a synchronous value() for hot paths. +export class CachedRedisNumber { + private readonly inner: CachedRedisValue<number>; + + constructor(options: CachedRedisNumberOptions) { + const min = options.min ?? Number.NEGATIVE_INFINITY; + const max = options.max ?? Number.POSITIVE_INFINITY; + const clamp = (n: number) => Math.min(max, Math.max(min, n)); + const fallback = clamp(options.defaultValue); + this.inner = new CachedRedisValue<number>({ + redis: options.redis, + key: options.key, + parse: (raw) => { + // Number("") is 0 (not NaN), so treat blank/whitespace as missing => fallback. + const n = raw == null || raw.trim() === "" ? Number.NaN : Number(raw); + return Number.isFinite(n) ? clamp(n) : fallback; + }, + defaultValue: fallback, + cacheTtlMs: options.cacheTtlMs, + logger: options.logger, + loggerName: "CachedRedisNumber", + }); + } + + value(): number { + return this.inner.get(); + } + + refresh(): Promise<number> { + return this.inner.refresh(); + } + + close(): Promise<void> { + return this.inner.close(); + } +} diff --git a/internal-packages/metrics-pipeline/src/consumer.test.ts b/internal-packages/metrics-pipeline/src/consumer.test.ts new file mode 100644 index 00000000000..672fa426999 --- /dev/null +++ b/internal-packages/metrics-pipeline/src/consumer.test.ts @@ -0,0 +1,392 @@ +import { createRedisClient } from "@internal/redis"; +import { redisTest } from "@internal/testcontainers"; +import { expect } from "vitest"; +import { CachedRedisFlag } from "./flag.js"; +import { CachedRedisNumber } from "./cachedValue.js"; +import { MetricsStreamConsumer } from "./consumer.js"; +import { MetricsStreamEmitter } from "./emitter.js"; +import { shardFor } from "./hash.js"; +import { streamKey, type MetricDefinition } from "./types.js"; + +async function waitFor(cond: () => boolean, timeoutMs = 5000): Promise<void> { + const start = Date.now(); + while (!cond()) { + if (Date.now() - start > timeoutMs) throw new Error("waitFor timed out"); + await new Promise((r) => setTimeout(r, 50)); + } +} + +function definitionFor(suffix: string, shardCount = 2): MetricDefinition { + return { name: `qm_${Date.now()}_${suffix}`, shardCount, consumerGroup: "cg", maxLen: 1000 }; +} + +redisTest( + "emitter -> consumer round trip maps rows, dedups, and acks", + async ({ redisOptions }) => { + const definition = definitionFor("rt"); + const emitter = new MetricsStreamEmitter({ + redis: redisOptions, + definition, + flag: { enabled: () => true }, + }); + const inserted: Array<{ rows: Array<Record<string, string>>; dedupToken: string }> = []; + + const consumer = new MetricsStreamConsumer<Record<string, string>>({ + redis: redisOptions, + definition, + consumerName: "c1", + mapEntry: (e) => ({ id: e.id, ...e.fields }), + insert: async (rows, { dedupToken }) => { + inserted.push({ rows, dedupToken }); + }, + blockMs: 200, + }); + + await consumer.start(); + emitter.emit("queueA", { op: "enqueue", q: "queueA" }); + emitter.emit("queueB", { op: "started", q: "queueB", wait: 42 }); + + await waitFor(() => inserted.flatMap((i) => i.rows).length >= 2); + await consumer.stop(); + + const rows = inserted.flatMap((i) => i.rows); + expect(rows).toContainEqual(expect.objectContaining({ op: "enqueue", q: "queueA" })); + expect(rows).toContainEqual( + expect.objectContaining({ op: "started", q: "queueB", wait: "42" }) + ); + expect(inserted[0]!.dedupToken).toMatch(/^[0-9a-f]{40}$/); + + const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined }); + for (const key of consumer.streamKeys()) { + const pending = (await admin.xpending(key, definition.consumerGroup)) as [ + number, + ...unknown[], + ]; + expect(pending[0]).toBe(0); + } + await admin.quit(); + await emitter.close(); + } +); + +redisTest("emit is a no-op when the flag is disabled", async ({ redisOptions }) => { + const definition = definitionFor("off"); + const emitter = new MetricsStreamEmitter({ + redis: redisOptions, + definition, + flag: { enabled: () => false }, + }); + + emitter.emit("q", { op: "enqueue", q: "q" }); + await new Promise((r) => setTimeout(r, 200)); + + const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined }); + const len = await admin.xlen(streamKey(definition, shardFor("q", definition.shardCount))); + expect(len).toBe(0); + await admin.quit(); + await emitter.close(); +}); + +redisTest("reclaims stale pending entries from a dead consumer", async ({ redisOptions }) => { + const definition = definitionFor("claim", 1); + const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined }); + const key = streamKey(definition, 0); + + await admin.xgroup("CREATE", key, definition.consumerGroup, "$", "MKSTREAM"); + await admin.xadd(key, "*", "op", "ack", "q", "qZ"); + await admin.xadd(key, "*", "op", "nack", "q", "qZ"); + await admin.xreadgroup( + "GROUP", + definition.consumerGroup, + "zombie", + "COUNT", + 10, + "STREAMS", + key, + ">" + ); + + const inserted: Array<Record<string, string>> = []; + const consumer = new MetricsStreamConsumer<Record<string, string>>({ + redis: redisOptions, + definition, + consumerName: "live", + mapEntry: (e) => ({ id: e.id, ...e.fields }), + insert: async (rows) => { + inserted.push(...rows); + }, + blockMs: 200, + claimIdleMs: 0, + }); + + await consumer.start(); + await waitFor(() => inserted.length >= 2); + await consumer.stop(); + + expect(inserted.map((r) => r.op).sort()).toEqual(["ack", "nack"]); + const pending = (await admin.xpending(key, definition.consumerGroup)) as [number, ...unknown[]]; + expect(pending[0]).toBe(0); + await admin.quit(); +}); + +redisTest( + "per-stream batches: one insert + distinct dedup token per shard stream", + async ({ redisOptions }) => { + const definition = definitionFor("pershard", 2); + const emitter = new MetricsStreamEmitter({ + redis: redisOptions, + definition, + flag: { enabled: () => true }, + }); + // Two shard keys that land on different shards. + const a = "shardkey-a"; + let b = "shardkey-b0"; + for (let i = 1; shardFor(b, 2) === shardFor(a, 2); i++) b = `shardkey-b${i}`; + + const inserted: Array<{ rows: Array<Record<string, string>>; dedupToken: string }> = []; + const consumer = new MetricsStreamConsumer<Record<string, string>>({ + redis: redisOptions, + definition, + consumerName: "c1", + mapEntry: (e) => ({ id: e.id, ...e.fields }), + insert: async (rows, { dedupToken }) => { + inserted.push({ rows, dedupToken }); + }, + blockMs: 200, + }); + + await consumer.start(); + emitter.emit(a, { op: "enqueue", q: a }); + emitter.emit(b, { op: "enqueue", q: b }); + await waitFor(() => inserted.flatMap((i) => i.rows).length >= 2); + await consumer.stop(); + await emitter.close(); + + // Each shard's batch is its own dedup block with its own (stream-scoped) token. + const batchesWithRows = inserted.filter((i) => i.rows.length > 0); + expect(batchesWithRows.length).toBe(2); + expect(new Set(batchesWithRows.map((i) => i.dedupToken)).size).toBe(2); + } +); + +redisTest( + "probe reports lag as null (not 0) when Redis cannot compute it", + async ({ redisOptions }) => { + const definition = definitionFor("nillag", 1); + const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined }); + const key = streamKey(definition, 0); + + await admin.xgroup("CREATE", key, definition.consumerGroup, "0", "MKSTREAM"); + const ids: string[] = []; + for (let i = 0; i < 5; i++) { + ids.push((await admin.xadd(key, "*", "op", "enqueue", "q", "qT")) as string); + } + // SETID to an arbitrary id makes the group's entries-read unknown => lag is nil + // (severe trimming can do the same in prod); the probe must NOT report that as 0. + await admin.xgroup("SETID", key, definition.consumerGroup, ids[2]!); + + const consumer = new MetricsStreamConsumer<Record<string, string>>({ + redis: redisOptions, + definition, + consumerName: "c1", + mapEntry: (e) => ({ id: e.id, ...e.fields }), + insert: async () => {}, + }); + try { + const states = await consumer.streamState(); + expect(states[0]!.lag).toBeNull(); + } finally { + await consumer.stop(); + await admin.quit(); + } + } +); + +redisTest( + "emitGauge XADDs an op=gauge snapshot onto the shared metrics stream", + async ({ redisOptions }) => { + const definition = definitionFor("gauge", 2); + const emitter = new MetricsStreamEmitter({ + redis: redisOptions, + definition, + flag: { enabled: () => true }, + }); + + // Emits before the connection is ready are dropped by design (loss-tolerant). + await emitter.waitUntilReady(); + emitter.emitGauge("q1", { + op: "gauge", + q: "q1", + ql: 5, + cc: 2, + lim: 10, + eql: 3, + ec: 1, + elim: 20, + thr: 0, + }); + + const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined }); + const key = streamKey(definition, shardFor("q1", 2)); + // Plain XADD (no odometer, no cum=0 seed) => exactly one entry, unlike counter emit(). + await waitFor2(async () => (await admin.xlen(key)) === 1); + const raw = (await admin.xrange(key, "-", "+")) as Array<[string, string[]]>; + const flat = raw[0]![1]; + const fields: Record<string, string> = {}; + for (let i = 0; i + 1 < flat.length; i += 2) fields[flat[i]!] = flat[i + 1]!; + expect(fields.op).toBe("gauge"); + expect(fields.q).toBe("q1"); + expect(fields.ql).toBe("5"); + expect(fields.thr).toBe("0"); + await admin.quit(); + await emitter.close(); + } +); + +async function waitFor2(cond: () => Promise<boolean>, timeoutMs = 5000): Promise<void> { + const start = Date.now(); + while (!(await cond())) { + if (Date.now() - start > timeoutMs) throw new Error("waitFor2 timed out"); + await new Promise((r) => setTimeout(r, 50)); + } +} + +redisTest("sampledSync gates on both the flag and the sample rate", async ({ redisOptions }) => { + const definition = definitionFor("sample"); + const off = new MetricsStreamEmitter({ + redis: redisOptions, + definition, + flag: { enabled: () => true }, + gaugeSampleRate: 0, + }); + const on = new MetricsStreamEmitter({ + redis: redisOptions, + definition, + flag: { enabled: () => true }, + gaugeSampleRate: 1, + }); + const disabled = new MetricsStreamEmitter({ + redis: redisOptions, + definition, + flag: { enabled: () => false }, + gaugeSampleRate: 1, + }); + + expect(off.sampledSync()).toBe(false); // rate 0 => never sampled in + expect(on.sampledSync()).toBe(true); // rate 1 + enabled => always + expect(disabled.sampledSync()).toBe(false); // disabled => never, regardless of rate + expect(on.enabledSync()).toBe(true); // enabledSync (counters) is unaffected by sampling + + await Promise.all([off.close(), on.close(), disabled.close()]); +}); + +redisTest("sampledSync honors a live rate provider (no reconstruct)", async ({ redisOptions }) => { + const definition = definitionFor("live"); + let rate = 1; + const emitter = new MetricsStreamEmitter({ + redis: redisOptions, + definition, + flag: { enabled: () => true }, + gaugeSampleRate: { value: () => rate }, + }); + expect(emitter.sampledSync()).toBe(true); + rate = 0; + expect(emitter.sampledSync()).toBe(false); + await emitter.close(); +}); + +redisTest("CachedRedisNumber reads live, clamps, and falls back", async ({ redisOptions }) => { + const key = `rate_${Date.now()}`; + const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined }); + const num = new CachedRedisNumber({ redis: redisOptions, key, defaultValue: 1, min: 0, max: 1 }); + + await num.refresh(); + expect(num.value()).toBe(1); // missing key => default + await admin.set(key, "0.25"); + await num.refresh(); + expect(num.value()).toBe(0.25); + await admin.set(key, "5"); + await num.refresh(); + expect(num.value()).toBe(1); // out of range => clamped + await admin.set(key, "nonsense"); + await num.refresh(); + expect(num.value()).toBe(1); // unparseable => default + + await num.close(); + await admin.quit(); +}); + +redisTest("streamState reports depth, lag, and pending per shard", async ({ redisOptions }) => { + const definition = definitionFor("state", 1); + const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined }); + const key = streamKey(definition, 0); + + await admin.xgroup("CREATE", key, definition.consumerGroup, "$", "MKSTREAM"); + await admin.xadd(key, "*", "op", "enqueue", "q", "qX"); + await admin.xadd(key, "*", "op", "ack", "q", "qX"); + // Read one entry as some consumer and leave it unacked -> 1 pending, 1 still undelivered. + await admin.xreadgroup( + "GROUP", + definition.consumerGroup, + "reader", + "COUNT", + 1, + "STREAMS", + key, + ">" + ); + + const consumer = new MetricsStreamConsumer<Record<string, string>>({ + redis: redisOptions, + definition, + consumerName: "c1", + mapEntry: (e) => ({ id: e.id, ...e.fields }), + insert: async () => {}, + }); + + try { + const states = await consumer.streamState(); + expect(states).toHaveLength(1); + expect(states[0]!.depth).toBe(2); + expect(states[0]!.pending).toBe(1); + expect(states[0]!.lag).toBe(1); + } finally { + await consumer.stop(); + await admin.quit(); + } +}); + +redisTest("CachedRedisFlag reads a redis key with caching", async ({ redisOptions }) => { + const key = `flag_${Date.now()}`; + const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined }); + const flag = new CachedRedisFlag({ redis: redisOptions, key, cacheTtlMs: 10_000 }); + + expect(flag.enabled()).toBe(false); + await flag.refresh(); + expect(flag.enabled()).toBe(false); + + await admin.set(key, "1"); + await flag.refresh(); + expect(flag.enabled()).toBe(true); + + await admin.set(key, "0"); + await flag.refresh(); + expect(flag.enabled()).toBe(false); + + await flag.close(); + await admin.quit(); +}); + +redisTest("CachedRedisFlag warms eagerly on construction", async ({ redisOptions }) => { + const key = `flag_eager_${Date.now()}`; + const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined }); + await admin.set(key, "1"); + + const flag = new CachedRedisFlag({ redis: redisOptions, key }); + // No manual refresh(): the constructor kicks one off so the first real read is warm. + await waitFor(() => flag.enabled() === true); + expect(flag.enabled()).toBe(true); + + await flag.close(); + await admin.quit(); +}); diff --git a/internal-packages/metrics-pipeline/src/consumer.ts b/internal-packages/metrics-pipeline/src/consumer.ts new file mode 100644 index 00000000000..9e333e70ab1 --- /dev/null +++ b/internal-packages/metrics-pipeline/src/consumer.ts @@ -0,0 +1,336 @@ +import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis"; +import { + getMeter, + type Counter, + type Histogram, + type Meter, + type ObservableGauge, + ValueType, +} from "@internal/tracing"; +import { Logger } from "@trigger.dev/core/logger"; +import { dedupTokenFromEntryIds } from "./idempotency.js"; +import { allStreamKeys, type MetricDefinition, type StreamEntry } from "./types.js"; + +export type MetricsStreamConsumerOptions<TRow> = { + redis: RedisOptions; + definition: MetricDefinition; + /** Unique per process; distinct replicas MUST use distinct names (PEL ownership). */ + consumerName: string; + /** Map a stream entry to a row, or null to drop it (still acked). */ + mapEntry: (entry: StreamEntry) => TRow | TRow[] | null; + /** Insert a batch. Must be idempotent w.r.t. dedupToken; throw to retry the batch. */ + insert: (rows: TRow[], opts: { dedupToken: string }) => Promise<void>; + batchSize?: number; + blockMs?: number; + claimIdleMs?: number; + /** How often to scan for stale pending entries (XAUTOCLAIM); not every poll. */ + reclaimIntervalMs?: number; + errorBackoffMs?: number; + logger?: Logger; + meter?: Meter; +}; + +type RawEntry = [id: string, fields: string[]]; +type RawStream = [key: string, entries: RawEntry[]]; + +/** Per-shard stream health, surfaced as observable gauges and usable directly in tests. + * `lag: null` means Redis could not compute it (entries trimmed past the group's read + * position) — treat as an alert, NOT as zero: it coincides with data loss. */ +export type ShardState = { shard: number; depth: number; lag: number | null; pending: number }; + +function parseFields(flat: string[]): Record<string, string> { + const out: Record<string, string> = {}; + for (let i = 0; i + 1 < flat.length; i += 2) { + out[flat[i]!] = flat[i + 1]!; + } + return out; +} + +const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)); + +/** + * Reads a sharded metrics stream via a consumer group, inserting each stream's poll-batch + * as its own dedup block (so an XAUTOCLAIM-reclaimed batch re-forms the same id set and + * token), acking only after a successful insert. Sequential read/insert/ack per process. + */ +export class MetricsStreamConsumer<TRow> { + private readonly redis: Redis; + private readonly probeRedis: Redis; + private readonly def: MetricDefinition; + private readonly keys: string[]; + private readonly consumerName: string; + private readonly batchSize: number; + private readonly blockMs: number; + private readonly claimIdleMs: number; + private readonly reclaimIntervalMs: number; + private lastReclaimAt = 0; + private readonly errorBackoffMs: number; + private readonly logger: Logger; + private readonly mapEntry: (entry: StreamEntry) => TRow | TRow[] | null; + private readonly insert: (rows: TRow[], opts: { dedupToken: string }) => Promise<void>; + + private readonly meter: Meter; + private readonly entriesCounter: Counter; + private readonly rowsCounter: Counter; + private readonly insertErrorCounter: Counter; + private readonly insertDuration: Histogram; + private readonly observables: ObservableGauge[]; + private readonly batchCallback: Parameters<Meter["addBatchObservableCallback"]>[0]; + + private running = false; + private loopPromise?: Promise<void>; + + constructor(options: MetricsStreamConsumerOptions<TRow>) { + this.logger = options.logger ?? new Logger("MetricsStreamConsumer", "info"); + const redisConfig = { ...options.redis, keyPrefix: undefined }; + this.redis = createRedisClient(redisConfig, { + onError: (error) => this.logger.error("consumer redis error", { error }), + }); + // Separate client so the observable-gauge probes never queue behind the blocking XREADGROUP. + this.probeRedis = createRedisClient(redisConfig, { + onError: (error) => this.logger.error("consumer probe redis error", { error }), + }); + this.def = options.definition; + this.keys = allStreamKeys(options.definition); + this.consumerName = options.consumerName; + this.batchSize = options.batchSize ?? 1000; + this.blockMs = options.blockMs ?? 1000; + this.claimIdleMs = options.claimIdleMs ?? 60_000; + this.reclaimIntervalMs = options.reclaimIntervalMs ?? 15_000; + this.errorBackoffMs = options.errorBackoffMs ?? 1000; + this.mapEntry = options.mapEntry; + this.insert = options.insert; + + this.meter = options.meter ?? getMeter("metrics-pipeline"); + this.entriesCounter = this.meter.createCounter("queue_metrics.consumer.entries", { + description: "Stream entries read (attr source=new|reclaimed)", + valueType: ValueType.INT, + }); + this.rowsCounter = this.meter.createCounter("queue_metrics.consumer.rows_inserted", { + description: "Rows inserted into the sink", + valueType: ValueType.INT, + }); + this.insertErrorCounter = this.meter.createCounter("queue_metrics.consumer.insert_errors", { + description: "Failed inserts (batch left pending for retry)", + valueType: ValueType.INT, + }); + this.insertDuration = this.meter.createHistogram("queue_metrics.consumer.insert_duration", { + description: "Sink insert latency", + unit: "ms", + valueType: ValueType.INT, + }); + + const depthGauge = this.meter.createObservableGauge("queue_metrics.consumer.stream_depth", { + description: "Entries currently in each shard stream (approaches MAXLEN => trimming)", + valueType: ValueType.INT, + }); + const lagGauge = this.meter.createObservableGauge("queue_metrics.consumer.group_lag", { + description: "Entries not yet delivered to the consumer group (consumer falling behind)", + valueType: ValueType.INT, + }); + const pendingGauge = this.meter.createObservableGauge("queue_metrics.consumer.pending", { + description: "Unacked (in-flight or stuck) entries in the group PEL", + valueType: ValueType.INT, + }); + const lagUnknownGauge = this.meter.createObservableGauge("queue_metrics.consumer.lag_unknown", { + description: + "1 when Redis cannot compute group lag (entries trimmed => data loss); alert on this", + valueType: ValueType.INT, + }); + this.observables = [depthGauge, lagGauge, pendingGauge, lagUnknownGauge]; + this.batchCallback = async (result) => { + const states = await this.streamState(); + for (const s of states) { + const attrs = { stream: this.def.name, shard: String(s.shard) }; + result.observe(depthGauge, s.depth, attrs); + if (s.lag !== null) result.observe(lagGauge, s.lag, attrs); + result.observe(lagUnknownGauge, s.lag === null ? 1 : 0, attrs); + result.observe(pendingGauge, s.pending, attrs); + } + }; + this.meter.addBatchObservableCallback(this.batchCallback, this.observables); + } + + async start(): Promise<void> { + if (this.running) return; + await this.ensureGroups(); + this.running = true; + this.loopPromise = this.loop(); + } + + async stop(): Promise<void> { + this.running = false; + this.meter.removeBatchObservableCallback(this.batchCallback, this.observables); + await this.loopPromise?.catch(() => {}); + await Promise.all([this.redis.quit().catch(() => {}), this.probeRedis.quit().catch(() => {})]); + } + + private async ensureGroups(): Promise<void> { + for (const key of this.keys) { + try { + // "0" (not "$"): a brand-new stream's group must not skip entries emitted + // between emitter boot and the first consumer's group creation. + await this.redis.xgroup("CREATE", key, this.def.consumerGroup, "0", "MKSTREAM"); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!message.includes("BUSYGROUP")) throw error; + } + } + } + + private async loop(): Promise<void> { + while (this.running) { + try { + if (Date.now() - this.lastReclaimAt >= this.reclaimIntervalMs) { + this.lastReclaimAt = Date.now(); + await this.reclaimStale(); + } + await this.readNew(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + // Self-heal a missing group (stream trimmed to nothing / deleted / Redis flushed): + // recreate it rather than wedging the loop on NOGROUP forever. + if (message.includes("NOGROUP")) { + this.logger.warn("consumer group missing; recreating", { error }); + await this.ensureGroups().catch(() => {}); + } else { + this.logger.error("consumer loop iteration failed", { error }); + } + await sleep(this.errorBackoffMs); + } + } + } + + private async readNew(): Promise<number> { + const ids = this.keys.map(() => ">"); + const response = (await this.redis.xreadgroup( + "GROUP", + this.def.consumerGroup, + this.consumerName, + "COUNT", + this.batchSize, + "BLOCK", + this.blockMs, + "STREAMS", + ...this.keys, + ...ids + )) as RawStream[] | null; + + if (!response) return 0; + return this.processStreams(response, "new"); + } + + private async reclaimStale(): Promise<void> { + for (const key of this.keys) { + const result = (await this.redis.xautoclaim( + key, + this.def.consumerGroup, + this.consumerName, + this.claimIdleMs, + "0", + "COUNT", + this.batchSize + )) as [string, RawEntry[], string[]] | null; + + const entries = result?.[1] ?? []; + if (entries.length === 0) continue; + await this.processStreams([[key, entries]], "reclaimed"); + } + } + + // One insert (dedup block) and XACK per stream, so a reclaimed batch re-forms the + // original per-stream id set and token. On insert failure that stream's entries stay + // pending for a later XAUTOCLAIM; other streams still progress. + private async processStreams(streams: RawStream[], source: "new" | "reclaimed"): Promise<number> { + let processed = 0; + let firstError: unknown; + + for (const [key, entries] of streams) { + if (entries.length === 0) continue; + const keyIds: string[] = []; + const rows: TRow[] = []; + for (const [id, flat] of entries) { + keyIds.push(id); + const mapped = this.mapEntry({ id, fields: parseFields(flat) }); + if (Array.isArray(mapped)) rows.push(...mapped); + else if (mapped !== null) rows.push(mapped); + } + this.entriesCounter.add(keyIds.length, { source }); + + if (rows.length > 0) { + const startedAt = Date.now(); + try { + await this.insert(rows, { dedupToken: dedupTokenFromEntryIds(keyIds, key) }); + } catch (error) { + this.insertErrorCounter.add(1); + firstError ??= error; + continue; + } finally { + this.insertDuration.record(Date.now() - startedAt); + } + this.rowsCounter.add(rows.length); + } + + await this.redis.xack(key, this.def.consumerGroup, ...keyIds); + processed += keyIds.length; + } + + if (firstError !== undefined) throw firstError; + return processed; + } + + /** Per-shard depth (XLEN), group lag, and pending — the consumer-health signals. */ + async streamState(): Promise<ShardState[]> { + return probeShardStates(this.probeRedis, this.keys, this.def.consumerGroup); + } + + /** All shard stream keys this consumer reads (for diagnostics/tests). */ + streamKeys(): string[] { + return this.keys.slice(); + } +} + +/** + * Per-shard depth/lag/pending for a metric stream — usable without a running consumer + * (e.g. from an admin route). `redis` should have keyPrefix unset, matching the stream keys. + */ +export async function probeShardStates( + redis: Redis, + keys: string[], + consumerGroup: string +): Promise<ShardState[]> { + const out: ShardState[] = []; + for (let shard = 0; shard < keys.length; shard++) { + const key = keys[shard]!; + const depth = Number(await redis.xlen(key)) || 0; + // lag defaults to null (unknown) and only becomes a number when the group is found and + // Redis reports one: a nil lag (or a missing group on an existing stream) means we can't + // compute it, e.g. entries were trimmed past the group's read position (data loss). + let lag: number | null = null; + let pending = 0; + try { + const groups = (await redis.call("XINFO", "GROUPS", key)) as unknown[]; + for (const raw of groups) { + const info = flatToMap(raw as unknown[]); + if (info.name === consumerGroup) { + const rawLag = info.lag; + lag = rawLag == null ? null : Number(rawLag); + if (lag !== null && !Number.isFinite(lag)) lag = null; + pending = Number(info.pending) || 0; + } + } + } catch { + // Stream/group may not exist yet; treat as zero. + } + out.push({ shard, depth, lag, pending }); + } + return out; +} + +function flatToMap(flat: unknown[]): Record<string, unknown> { + const out: Record<string, unknown> = {}; + for (let i = 0; i + 1 < flat.length; i += 2) { + out[String(flat[i])] = flat[i + 1]; + } + return out; +} diff --git a/internal-packages/metrics-pipeline/src/emitter.ts b/internal-packages/metrics-pipeline/src/emitter.ts new file mode 100644 index 00000000000..692956d98cb --- /dev/null +++ b/internal-packages/metrics-pipeline/src/emitter.ts @@ -0,0 +1,242 @@ +import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis"; +import { getMeter, type Counter, type Meter, ValueType } from "@internal/tracing"; +import { Logger } from "@trigger.dev/core/logger"; +import { shardFor } from "./hash.js"; +import { streamKey, type MetricDefinition, type MetricFields } from "./types.js"; + +export type MetricsStreamEmitterOptions = { + redis: RedisOptions; + definition: MetricDefinition; + /** Synchronous enabled check (e.g. CachedRedisFlag); emits are no-ops when false. */ + flag: { enabled(): boolean }; + /** Probability (0..1) that a sampled emission fires; applies to `sampledSync()`, not + * `emit()`. Pass a `{ value() }` provider (e.g. CachedRedisNumber) to tune it live + * without a redeploy. Default 1 (always). */ + gaugeSampleRate?: number | { value(): number }; + /** TTL (ms) refreshed on every counter write on the per-(queue,op) odometer key. + * Active queues never expire; idle-past-TTL queues purge and self-heal on return. + * Default 7 days. */ + counterOdometerTtlMs?: number; + /** TTL (ms) for per-concurrency-key odometers; short because key cardinality is + * user-controlled and cumulative counters make idle-gap expiry loss-free. Default 24h. */ + ckOdometerTtlMs?: number; + logger?: Logger; + meter?: Meter; +}; + +type CumulativeCommand = ( + odometerKey: string, + streamKey: string, + ttlMs: string, + maxLen: string, + op: string, + q: string, + ...extraFields: string[] +) => Promise<unknown>; + +type CumulativeCkCommand = ( + odometerKey: string, + ckOdometerKey: string, + streamKey: string, + ttlMs: string, + ckTtlMs: string, + maxLen: string, + op: string, + q: string, + ck: string, + ...extraFields: string[] +) => Promise<unknown>; + +// INCR the odometer, refresh its TTL, and XADD the reading (new value as `cum`) in one round +// trip. Refresh-on-write is load-bearing: only genuinely idle queues expire. On first creation +// (v==1) XADD a cum=0 baseline first (smaller stream id => sorts first) so deltaSum captures the +// 0->1 transition and the total reconstructs exactly. +// ARGV: [1]=ttlMs [2]=maxLen [3]=op [4]=q [5..]=extra field/value pairs (e.g. wait). +const CUMULATIVE_LUA = ` +local v = redis.call('INCR', KEYS[1]) +redis.call('PEXPIRE', KEYS[1], ARGV[1]) +local maxlen = tonumber(ARGV[2]) or 0 +local function xadd(cum, withExtra) + local x = {'XADD', KEYS[2]} + if maxlen > 0 then x[#x+1]='MAXLEN'; x[#x+1]='~'; x[#x+1]=ARGV[2] end + x[#x+1]='*' + x[#x+1]='op'; x[#x+1]=ARGV[3] + x[#x+1]='q'; x[#x+1]=ARGV[4] + if withExtra then for i=5,#ARGV do x[#x+1]=ARGV[i] end end + x[#x+1]='cum'; x[#x+1]=cum + redis.call(unpack(x)) +end +if v == 1 then xadd(0, false) end +xadd(v, true) +`; + +// CK variant: advances base + per-key odometers, ONE reading entry carries both (cum + +// ck/ckcum), so per-key attribution adds no stream volume. Baselines seed independently: +// cum-only entry = base row, ck+ckcum-only entry = per-key row, reading entry = both. +// KEYS: [1]=baseOdometer [2]=ckOdometer [3]=stream. ARGV: [1]=baseTtlMs [2]=ckTtlMs +// [3]=maxLen [4]=op [5]=q [6]=ck [7..]=extra field/value pairs. +const CUMULATIVE_CK_LUA = ` +local v = redis.call('INCR', KEYS[1]) +redis.call('PEXPIRE', KEYS[1], ARGV[1]) +local ckv = redis.call('INCR', KEYS[2]) +redis.call('PEXPIRE', KEYS[2], ARGV[2]) +local maxlen = tonumber(ARGV[3]) or 0 +local function xadd(fields, withExtra) + local x = {'XADD', KEYS[3]} + if maxlen > 0 then x[#x+1]='MAXLEN'; x[#x+1]='~'; x[#x+1]=ARGV[3] end + x[#x+1]='*' + x[#x+1]='op'; x[#x+1]=ARGV[4] + x[#x+1]='q'; x[#x+1]=ARGV[5] + if withExtra then for i=7,#ARGV do x[#x+1]=ARGV[i] end end + for i=1,#fields do x[#x+1]=fields[i] end + redis.call(unpack(x)) +end +if v == 1 then xadd({'cum', 0}, false) end +if ckv == 1 then xadd({'ck', ARGV[6], 'ckcum', 0}, false) end +xadd({'ck', ARGV[6], 'cum', v, 'ckcum', ckv}, true) +`; + +/** Node-side producer: XADDs events to a sharded metrics stream, gated on a flag. */ +export class MetricsStreamEmitter { + private readonly redis: Redis; + private readonly def: MetricDefinition; + private readonly flag: { enabled(): boolean }; + private readonly sampleRate: () => number; + private readonly odometerTtlMs: number; + private readonly ckOdometerTtlMs: number; + private readonly logger: Logger; + private readonly emittedCounter: Counter; + private readonly errorCounter: Counter; + + constructor(options: MetricsStreamEmitterOptions) { + this.logger = options.logger ?? new Logger("MetricsStreamEmitter", "warn"); + this.redis = createRedisClient( + { ...options.redis, keyPrefix: undefined }, + { onError: (error) => this.logger.error("emitter redis error", { error }) } + ); + this.redis.defineCommand("qmEmitCumulative", { numberOfKeys: 2, lua: CUMULATIVE_LUA }); + this.redis.defineCommand("qmEmitCumulativeCk", { numberOfKeys: 3, lua: CUMULATIVE_CK_LUA }); + this.odometerTtlMs = options.counterOdometerTtlMs ?? 7 * 24 * 60 * 60 * 1000; + this.ckOdometerTtlMs = options.ckOdometerTtlMs ?? 24 * 60 * 60 * 1000; + this.def = options.definition; + this.flag = options.flag; + const rate = options.gaugeSampleRate; + if (typeof rate === "object") { + this.sampleRate = () => rate.value(); + } else { + const fixed = Math.min(1, Math.max(0, rate ?? 1)); + this.sampleRate = () => fixed; + } + + const meter = options.meter ?? getMeter("metrics-pipeline"); + this.emittedCounter = meter.createCounter("queue_metrics.emitter.emitted", { + description: "Node-side metric events XADDed to the stream", + valueType: ValueType.INT, + }); + this.errorCounter = meter.createCounter("queue_metrics.emitter.errors", { + description: "Failed metric-event XADDs (dropped)", + valueType: ValueType.INT, + }); + } + + enabledSync(): boolean { + return this.flag.enabled(); + } + + // Enabled AND (probabilistically) sampled-in. For high-frequency sampled emissions + // (e.g. Lua gauges); exact-count events use enabledSync()/emit() and are never sampled. + sampledSync(): boolean { + if (!this.flag.enabled()) return false; + const rate = this.sampleRate(); + if (rate >= 1) return true; + if (rate <= 0) return false; + return Math.random() < rate; + } + + // Fire-and-forget gauge emit: a plain XADD of an op=gauge snapshot (no odometer). The + // gauge value was read atomically inside the queue op's Lua and returned on the reply; + // this just lands it on the metrics stream. Loss-tolerant (sampled), never throws into + // the caller. Shares the counter stream (one stream family on the metrics Redis). + emitGauge(shardKey: string, fields: MetricFields): void { + if (!this.flag.enabled()) return; + // Drop rather than queue while the metrics Redis is unreachable: ioredis would hold + // every command in its offline queue until rejection, and metrics are loss-tolerant. + if (this.redis.status !== "ready") return; + const op = String(fields.op ?? "gauge"); + const stream = streamKey(this.def, shardFor(shardKey, this.def.shardCount)); + const args: string[] = []; + if (this.def.maxLen) args.push("MAXLEN", "~", String(this.def.maxLen)); + args.push("*"); + for (const [field, value] of Object.entries(fields)) { + args.push(field, String(value)); + } + this.emittedCounter.add(1, { op }); + this.redis.xadd(stream, ...(args as [string, ...string[]])).catch((error) => { + this.errorCounter.add(1); + this.logger.debug("metrics gauge emit failed", { error, stream }); + }); + } + + // Fire-and-forget cumulative counter emit: advances the per-(queue,op) odometer and + // XADDs its new absolute value. No-op when disabled, never throws into the caller. A + // lost XADD self-heals (the next reading restates the total); the INCR is never sampled. + // A non-empty `fields.ck` also advances a per-concurrency-key odometer and rides the + // same entry as ck/ckcum (see CUMULATIVE_CK_LUA for the baseline/row mapping). + emit(shardKey: string, fields: MetricFields): void { + if (!this.flag.enabled()) return; + if (this.redis.status !== "ready") return; + const op = String(fields.op ?? "unknown"); + const q = String(fields.q ?? ""); + const ck = fields.ck != null && String(fields.ck) !== "" ? String(fields.ck) : null; + const shard = shardFor(shardKey, this.def.shardCount); + const stream = streamKey(this.def, shard); + // The odometer carries the stream's {shard} hash tag so INCR + XADD stay in one + // Cluster slot (the shard is derived from the queue, so the mapping is stable). + // The key format is part of the rolling-deploy data shape: concurrent old/new + // emitters with different formats split an odometer and corrupt its deltas. + const odometerKey = `${this.def.name}_cum:{${shard}}:${op}:${q}`; + const extra: string[] = []; + for (const [field, value] of Object.entries(fields)) { + if (field === "op" || field === "q" || field === "ck") continue; + extra.push(field, String(value)); + } + this.emittedCounter.add(1, { op }); + const maxLen = String(this.def.maxLen ?? 0); + const done = (error: unknown) => { + this.errorCounter.add(1); + this.logger.debug("metrics emit failed", { error, stream }); + }; + if (ck) { + const client = this.redis as unknown as { qmEmitCumulativeCk: CumulativeCkCommand }; + client + .qmEmitCumulativeCk( + odometerKey, + `${odometerKey}:ck:${ck}`, + stream, + String(this.odometerTtlMs), + String(this.ckOdometerTtlMs), + maxLen, + op, + q, + ck, + ...extra + ) + .catch(done); + return; + } + const client = this.redis as unknown as { qmEmitCumulative: CumulativeCommand }; + client + .qmEmitCumulative(odometerKey, stream, String(this.odometerTtlMs), maxLen, op, q, ...extra) + .catch(done); + } + + // Resolves once the metrics Redis connection is ready (emits before that are dropped). + waitUntilReady(): Promise<void> { + if (this.redis.status === "ready") return Promise.resolve(); + return new Promise((resolve) => this.redis.once("ready", () => resolve())); + } + + async close(): Promise<void> { + await this.redis.quit(); + } +} diff --git a/internal-packages/metrics-pipeline/src/flag.ts b/internal-packages/metrics-pipeline/src/flag.ts new file mode 100644 index 00000000000..5931e088939 --- /dev/null +++ b/internal-packages/metrics-pipeline/src/flag.ts @@ -0,0 +1,46 @@ +import type { RedisOptions } from "@internal/redis"; +import type { Logger } from "@trigger.dev/core/logger"; +import { CachedRedisValue } from "./cachedValue.js"; + +export type CachedRedisFlagOptions = { + redis: RedisOptions; + /** Redis key holding the flag. A value of "1"/"true"/"on"/"enabled" is truthy. */ + key: string; + cacheTtlMs?: number; + defaultValue?: boolean; + logger?: Logger; +}; + +const TRUTHY = new Set(["1", "true", "on", "enabled", "yes"]); + +/** + * Boolean feature flag from a Redis key with a short stale-while-revalidate cache, + * exposing a synchronous getter for hot paths (building Lua ARGV on every op). + */ +export class CachedRedisFlag { + private readonly inner: CachedRedisValue<boolean>; + + constructor(options: CachedRedisFlagOptions) { + this.inner = new CachedRedisValue<boolean>({ + redis: options.redis, + key: options.key, + parse: (raw) => raw != null && TRUTHY.has(raw.trim().toLowerCase()), + defaultValue: options.defaultValue ?? false, + cacheTtlMs: options.cacheTtlMs, + logger: options.logger, + loggerName: "CachedRedisFlag", + }); + } + + enabled(): boolean { + return this.inner.get(); + } + + refresh(): Promise<boolean> { + return this.inner.refresh(); + } + + async close(): Promise<void> { + await this.inner.close(); + } +} diff --git a/internal-packages/metrics-pipeline/src/hash.ts b/internal-packages/metrics-pipeline/src/hash.ts new file mode 100644 index 00000000000..b14324c138a --- /dev/null +++ b/internal-packages/metrics-pipeline/src/hash.ts @@ -0,0 +1,15 @@ +/** FNV-1a 32-bit hash. Deterministic across processes; used only for sharding. */ +export function fnv1a32(str: string): number { + let hash = 0x811c9dc5; + for (let i = 0; i < str.length; i++) { + hash ^= str.charCodeAt(i); + hash = Math.imul(hash, 0x01000193); + } + return hash >>> 0; +} + +/** Deterministic shard index in [0, shardCount) for a key. */ +export function shardFor(key: string, shardCount: number): number { + if (shardCount <= 1) return 0; + return fnv1a32(key) % shardCount; +} diff --git a/internal-packages/metrics-pipeline/src/idempotency.ts b/internal-packages/metrics-pipeline/src/idempotency.ts new file mode 100644 index 00000000000..60cbd661f53 --- /dev/null +++ b/internal-packages/metrics-pipeline/src/idempotency.ts @@ -0,0 +1,11 @@ +import { createHash } from "node:crypto"; + +// Deterministic, order-independent token over a batch of entry ids. A redelivered +// batch yields the same token, so ClickHouse's raw-table dedup window drops the replay. +// `scope` (the stream key) disambiguates id sets that could collide across streams. +export function dedupTokenFromEntryIds(ids: string[], scope = ""): string { + const sorted = [...ids].sort(); + return createHash("sha1") + .update(`${scope}|${sorted.join(",")}`) + .digest("hex"); +} diff --git a/internal-packages/metrics-pipeline/src/index.ts b/internal-packages/metrics-pipeline/src/index.ts new file mode 100644 index 00000000000..223c5feab17 --- /dev/null +++ b/internal-packages/metrics-pipeline/src/index.ts @@ -0,0 +1,26 @@ +export { CachedRedisFlag, type CachedRedisFlagOptions } from "./flag.js"; +export { + CachedRedisNumber, + type CachedRedisNumberOptions, + CachedRedisValue, + type CachedRedisValueOptions, +} from "./cachedValue.js"; +export { MetricsStreamEmitter, type MetricsStreamEmitterOptions } from "./emitter.js"; +export { + MetricsStreamConsumer, + type MetricsStreamConsumerOptions, + type ShardState, + probeShardStates, +} from "./consumer.js"; +export { createMetricsGaugeComputeLua, type GaugeComputeLuaParams } from "./lua.js"; +export { dedupTokenFromEntryIds } from "./idempotency.js"; +export { shardFor, fnv1a32 } from "./hash.js"; +export { + streamKey, + allStreamKeys, + entryTimeMs, + entryOrderKey, + type MetricDefinition, + type MetricFields, + type StreamEntry, +} from "./types.js"; diff --git a/internal-packages/metrics-pipeline/src/lua.ts b/internal-packages/metrics-pipeline/src/lua.ts new file mode 100644 index 00000000000..64f3b896c0d --- /dev/null +++ b/internal-packages/metrics-pipeline/src/lua.ts @@ -0,0 +1,50 @@ +// Each field is a Lua expression evaluated inside the target script. queueLimit/ +// envLimit must be the EFFECTIVE enforced limit, else an unset limit reads as throttled. +export type GaugeComputeLuaParams = { + // Lua boolean expression; when true the gauge is computed (else the extra reads are skipped). + enabledArg: string; + queued: string; + running: string; + queueLimit: string; + envQueued: string; + envRunning: string; + envLimit: string; + // Lua statements run first inside the pcall (e.g. to compute aggregate locals). + preamble?: string; + // Lua boolean expression (in __cc/__lim/__ql) for the throttled flag. Pass "false" + // where cc >= lim is not a valid throttle signal (e.g. summed CK aggregates). + throttledExpr?: string; + // CK-health extras (both or neither): appended as an optional gauge tail, gauge[8]/gauge[9]. + ckBacklogged?: string; + ckMaxWaitMs?: string; +}; + +// Computes an op=gauge snapshot into the enclosing script's `__qm_g` local (a flat +// {ql, cc, lim, eql, ec, elim, thr} array) so the script can RETURN it; Node then XADDs it +// to the metrics Redis. No Redis write here (the run-queue Redis carries no metrics stream). +// Gated on the sample flag and pcall-wrapped. The script MUST declare `local __qm_g` first. +export function createMetricsGaugeComputeLua(params: GaugeComputeLuaParams): string { + const throttled = params.throttledExpr ?? "__cc >= __lim and __ql > 0"; + const hasCk = params.ckBacklogged != null && params.ckMaxWaitMs != null; + const gauge = hasCk + ? ` local __ckq = tonumber(${params.ckBacklogged}) or 0 + local __ckw = tonumber(${params.ckMaxWaitMs}) or 0 + __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr, __ckq, __ckw}` + : ` __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr}`; + + return ` +if ${params.enabledArg} then + pcall(function() + ${params.preamble ?? ""} + local __ql = tonumber(${params.queued}) or 0 + local __cc = tonumber(${params.running}) or 0 + local __lim = tonumber(${params.queueLimit}) or 0 + local __eql = tonumber(${params.envQueued}) or 0 + local __ec = tonumber(${params.envRunning}) or 0 + local __elim = tonumber(${params.envLimit}) or 0 + local __thr = 0 + if ${throttled} then __thr = 1 end +${gauge} + end) +end`; +} diff --git a/internal-packages/metrics-pipeline/src/pipeline.test.ts b/internal-packages/metrics-pipeline/src/pipeline.test.ts new file mode 100644 index 00000000000..73979310798 --- /dev/null +++ b/internal-packages/metrics-pipeline/src/pipeline.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it } from "vitest"; +import { createMetricsGaugeComputeLua } from "./lua.js"; +import { dedupTokenFromEntryIds } from "./idempotency.js"; +import { fnv1a32, shardFor } from "./hash.js"; +import { allStreamKeys, entryOrderKey, entryTimeMs, streamKey } from "./types.js"; + +describe("shardFor", () => { + it("is deterministic and in range", () => { + expect(shardFor("queueA", 1)).toBe(0); + const s = shardFor("queueA", 4); + expect(s).toBeGreaterThanOrEqual(0); + expect(s).toBeLessThan(4); + expect(shardFor("queueA", 4)).toBe(s); + expect(fnv1a32("queueA")).toBe(fnv1a32("queueA")); + }); +}); + +describe("dedupTokenFromEntryIds", () => { + it("is order-independent and set-sensitive", () => { + expect(dedupTokenFromEntryIds(["1-0", "2-0"])).toBe(dedupTokenFromEntryIds(["2-0", "1-0"])); + expect(dedupTokenFromEntryIds(["1-0"])).not.toBe(dedupTokenFromEntryIds(["2-0"])); + expect(dedupTokenFromEntryIds(["1-0"])).toMatch(/^[0-9a-f]{40}$/); + }); +}); + +describe("stream keys", () => { + it("names and parses entry time", () => { + expect(streamKey({ name: "queue_metrics" }, 3)).toBe("queue_metrics:{3}"); + expect(allStreamKeys({ name: "qm", shardCount: 2, consumerGroup: "cg" })).toEqual([ + "qm:{0}", + "qm:{1}", + ]); + expect(entryTimeMs("1717000000000-5")).toBe(1717000000000); + expect(entryTimeMs("nope")).toBeNull(); + }); + + it("entryOrderKey stays exact and strictly monotonic at real epoch magnitudes", () => { + const ms = 1783000000000; // ~2026: ms*1e6 is past JS safe-integer range, so a number key + const k = (seq: number) => BigInt(entryOrderKey(`${ms}-${seq}`)); + // adjacent seq within one ms must not collapse to the same key (the float bug) + expect(k(0)).toBe(BigInt(ms) * 1000000n); + expect(k(1) - k(0)).toBe(1n); + expect(k(2) - k(1)).toBe(1n); + // a later ms always outranks any seq of an earlier ms (up to the 1M/ms factor) + expect(BigInt(entryOrderKey(`${ms + 1}-0`))).toBeGreaterThan(k(999999)); + }); +}); + +describe("createMetricsGaugeComputeLua", () => { + it("assigns __qm_g inside a gated, pcall-wrapped block and never XADDs", () => { + const lua = createMetricsGaugeComputeLua({ + enabledArg: "ARGV[#ARGV] == '1'", + queued: "redis.call('ZCARD', KEYS[2])", + running: "queueCurrent", + queueLimit: "queueLimit", + envQueued: "redis.call('ZCARD', KEYS[8])", + envRunning: "envCurrent", + envLimit: "envLimit", + }); + + expect(lua).toContain("if ARGV[#ARGV] == '1' then"); + expect(lua).toContain("pcall(function()"); + expect(lua).toContain("__qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr}"); + expect(lua).toContain("if __cc >= __lim and __ql > 0 then __thr = 1 end"); + // The whole point of the refactor: no Redis write happens in the run-queue script. + expect(lua).not.toContain("XADD"); + }); + + it("honors a custom throttled expression and preamble", () => { + const lua = createMetricsGaugeComputeLua({ + enabledArg: "true", + preamble: "local agg = 1", + queued: "0", + running: "0", + queueLimit: "0", + envQueued: "0", + envRunning: "0", + envLimit: "0", + throttledExpr: "false", + }); + expect(lua).toContain("local agg = 1"); + expect(lua).toContain("if false then __thr = 1 end"); + expect(lua).not.toContain("XADD"); + }); + + it("appends the CK-health tail only when both CK params are set", () => { + const withCk = createMetricsGaugeComputeLua({ + enabledArg: "true", + queued: "0", + running: "0", + queueLimit: "0", + envQueued: "0", + envRunning: "0", + envLimit: "0", + ckBacklogged: "redis.call('ZCARD', ckIndexKey)", + ckMaxWaitMs: "__ckwait", + }); + expect(withCk).toContain( + "__qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr, __ckq, __ckw}" + ); + expect(withCk).toContain("local __ckq = tonumber(redis.call('ZCARD', ckIndexKey)) or 0"); + + const withoutCk = createMetricsGaugeComputeLua({ + enabledArg: "true", + queued: "0", + running: "0", + queueLimit: "0", + envQueued: "0", + envRunning: "0", + envLimit: "0", + ckBacklogged: "0", + }); + expect(withoutCk).toContain("__qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr}"); + expect(withoutCk).not.toContain("__ckq"); + }); +}); diff --git a/internal-packages/metrics-pipeline/src/types.ts b/internal-packages/metrics-pipeline/src/types.ts new file mode 100644 index 00000000000..d9e9e43f554 --- /dev/null +++ b/internal-packages/metrics-pipeline/src/types.ts @@ -0,0 +1,42 @@ +export type MetricFields = Record<string, string | number>; + +export type StreamEntry = { + id: string; + fields: Record<string, string>; +}; + +export type MetricDefinition = { + /** Logical name, e.g. "queue_metrics". Used as the stream key prefix. */ + name: string; + shardCount: number; + consumerGroup: string; + /** Approximate MAXLEN cap applied on XADD (`MAXLEN ~ N`). Omit for unbounded. */ + maxLen?: number; +}; + +// Keys are used verbatim on every access path (Lua ARGV, emitter, consumer), so +// they must NOT be subject to an ioredis keyPrefix. `{shard}` is a Cluster hash tag. +export function streamKey(definition: Pick<MetricDefinition, "name">, shard: number): string { + return `${definition.name}:{${shard}}`; +} + +export function allStreamKeys(definition: MetricDefinition): string[] { + return Array.from({ length: Math.max(1, definition.shardCount) }, (_, shard) => + streamKey(definition, shard) + ); +} + +// The ms part of a stream entry id is its emission time. +export function entryTimeMs(id: string): number | null { + const ms = Number(id.split("-")[0]); + return Number.isFinite(ms) ? ms : null; +} + +// Ordering key from a stream id (`<ms>-<seq>`) = ms*1e6+seq, for deltaSumTimestamp. BigInt + +// string because ms*1e6 exceeds JS safe-integer range at real epoch magnitudes (a number would +// collapse nearby seq values); the ClickHouse order_key column is UInt64 and takes the string. +// The 1e6 factor (1M entries/ms/shard, far above any single Redis stream) stays within UInt64. +export function entryOrderKey(id: string): string { + const [ms, seq] = id.split("-"); + return (BigInt(Number(ms) || 0) * 1000000n + BigInt(Number(seq) || 0)).toString(); +} diff --git a/internal-packages/metrics-pipeline/test/setup.ts b/internal-packages/metrics-pipeline/test/setup.ts new file mode 100644 index 00000000000..b2bacd6baf5 --- /dev/null +++ b/internal-packages/metrics-pipeline/test/setup.ts @@ -0,0 +1,4 @@ +import { vi } from "vitest"; + +// Set extended timeout for container tests +vi.setConfig({ testTimeout: 60_000 }); diff --git a/internal-packages/metrics-pipeline/tsconfig.build.json b/internal-packages/metrics-pipeline/tsconfig.build.json new file mode 100644 index 00000000000..89c87a3dc67 --- /dev/null +++ b/internal-packages/metrics-pipeline/tsconfig.build.json @@ -0,0 +1,21 @@ +{ + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts"], + "compilerOptions": { + "composite": true, + "target": "ES2020", + "lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"], + "outDir": "dist", + "module": "Node16", + "moduleResolution": "Node16", + "moduleDetection": "force", + "verbatimModuleSyntax": false, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "preserveWatchOutput": true, + "skipLibCheck": true, + "strict": true, + "declaration": true + } +} diff --git a/internal-packages/metrics-pipeline/tsconfig.json b/internal-packages/metrics-pipeline/tsconfig.json new file mode 100644 index 00000000000..af630abe1f1 --- /dev/null +++ b/internal-packages/metrics-pipeline/tsconfig.json @@ -0,0 +1,8 @@ +{ + "references": [{ "path": "./tsconfig.src.json" }, { "path": "./tsconfig.test.json" }], + "compilerOptions": { + "moduleResolution": "Node16", + "module": "Node16", + "customConditions": ["@triggerdotdev/source"] + } +} diff --git a/internal-packages/metrics-pipeline/tsconfig.src.json b/internal-packages/metrics-pipeline/tsconfig.src.json new file mode 100644 index 00000000000..0df3d2d222f --- /dev/null +++ b/internal-packages/metrics-pipeline/tsconfig.src.json @@ -0,0 +1,20 @@ +{ + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "src/**/*.test.ts"], + "compilerOptions": { + "composite": true, + "target": "ES2020", + "lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"], + "module": "Node16", + "moduleResolution": "Node16", + "moduleDetection": "force", + "verbatimModuleSyntax": false, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "preserveWatchOutput": true, + "skipLibCheck": true, + "strict": true, + "customConditions": ["@triggerdotdev/source"] + } +} diff --git a/internal-packages/metrics-pipeline/tsconfig.test.json b/internal-packages/metrics-pipeline/tsconfig.test.json new file mode 100644 index 00000000000..4c06c9f57bb --- /dev/null +++ b/internal-packages/metrics-pipeline/tsconfig.test.json @@ -0,0 +1,21 @@ +{ + "include": ["src/**/*.test.ts"], + "references": [{ "path": "./tsconfig.src.json" }], + "compilerOptions": { + "composite": true, + "target": "ES2020", + "lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"], + "module": "Node16", + "moduleResolution": "Node16", + "moduleDetection": "force", + "verbatimModuleSyntax": false, + "types": ["vitest/globals"], + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "preserveWatchOutput": true, + "skipLibCheck": true, + "strict": true, + "customConditions": ["@triggerdotdev/source"] + } +} diff --git a/internal-packages/metrics-pipeline/vitest.config.ts b/internal-packages/metrics-pipeline/vitest.config.ts new file mode 100644 index 00000000000..daafd294fa8 --- /dev/null +++ b/internal-packages/metrics-pipeline/vitest.config.ts @@ -0,0 +1,17 @@ +import { defineConfig } from "vitest/config"; +import { DurationShardingSequencer } from "@internal/testcontainers/sequencer"; + +export default defineConfig({ + test: { + sequence: { sequencer: DurationShardingSequencer }, + globals: true, + retry: process.env.CI ? 2 : 0, + environment: "node", + setupFiles: ["./test/setup.ts"], + testTimeout: 30000, + hookTimeout: 30000, + }, + esbuild: { + target: "node18", + }, +}); diff --git a/internal-packages/run-engine/package.json b/internal-packages/run-engine/package.json index 8d53974d10b..516e6a18696 100644 --- a/internal-packages/run-engine/package.json +++ b/internal-packages/run-engine/package.json @@ -21,6 +21,7 @@ }, "dependencies": { "@internal/redis": "workspace:*", + "@internal/metrics-pipeline": "workspace:*", "@internal/run-store": "workspace:*", "@trigger.dev/redis-worker": "workspace:*", "@internal/tracing": "workspace:*", diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 3c1f4330a0f..b5131766559 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -218,6 +218,7 @@ export class RunEngine { callback: this.#concurrencySweeperCallback.bind(this), }, shardCount: options.queue?.shardCount, + queueMetrics: options.queue?.queueMetrics, masterQueueConsumersDisabled: options.queue?.masterQueueConsumersDisabled, masterQueueConsumersIntervalMs: options.queue?.masterQueueConsumersIntervalMs, processWorkerQueueDebounceMs: options.queue?.processWorkerQueueDebounceMs, @@ -1168,6 +1169,7 @@ export class RunEngine { tx: prisma, skipRunLock: true, includeTtl: true, + anchorEligibilityAtQueuePosition: true, enableFastPath, }); } catch (enqueueError) { @@ -1611,9 +1613,26 @@ export class RunEngine { async lengthOfQueue( environment: MinimalAuthenticatedEnvironment, - queue: string + queue: string, + concurrencyKey?: string + ): Promise<number> { + return this.runQueue.lengthOfQueue(environment, queue, concurrencyKey); + } + + async currentConcurrencyOfQueue( + environment: MinimalAuthenticatedEnvironment, + queue: string, + concurrencyKey?: string ): Promise<number> { - return this.runQueue.lengthOfQueue(environment, queue); + return this.runQueue.currentConcurrencyOfQueue(environment, queue, concurrencyKey); + } + + async oldestMessageInQueue( + environment: MinimalAuthenticatedEnvironment, + queue: string, + concurrencyKey?: string + ): Promise<number | undefined> { + return this.runQueue.oldestMessageInQueue(environment, queue, concurrencyKey); } async concurrencyOfEnvQueue(environment: MinimalAuthenticatedEnvironment): Promise<number> { @@ -1634,6 +1653,22 @@ export class RunEngine { return this.runQueue.currentConcurrencyOfQueues(environment, queues); } + async concurrencyKeyBreakdown( + environment: MinimalAuthenticatedEnvironment, + queue: string, + options?: { limit?: number } + ) { + return this.runQueue.concurrencyKeyBreakdown(environment, queue, options); + } + + async concurrencyKeyLiveStats( + environment: MinimalAuthenticatedEnvironment, + queue: string, + concurrencyKeys: string[] + ) { + return this.runQueue.concurrencyKeyLiveStats(environment, queue, concurrencyKeys); + } + async removeEnvironmentQueuesFromMasterQueue({ runtimeEnvironmentId, organizationId, diff --git a/internal-packages/run-engine/src/engine/systems/delayedRunSystem.ts b/internal-packages/run-engine/src/engine/systems/delayedRunSystem.ts index 68939a2d874..d0d603f9f02 100644 --- a/internal-packages/run-engine/src/engine/systems/delayedRunSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/delayedRunSystem.ts @@ -164,6 +164,7 @@ export class DelayedRunSystem { batchId: run.batchId ?? undefined, skipRunLock: true, includeTtl: true, + anchorEligibilityAtQueuePosition: true, }); const queuedAt = new Date(); diff --git a/internal-packages/run-engine/src/engine/systems/enqueueSystem.ts b/internal-packages/run-engine/src/engine/systems/enqueueSystem.ts index dc9d029c38c..2b5aac6d376 100644 --- a/internal-packages/run-engine/src/engine/systems/enqueueSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/enqueueSystem.ts @@ -37,6 +37,7 @@ export class EnqueueSystem { runnerId, skipRunLock, includeTtl = false, + anchorEligibilityAtQueuePosition = false, enableFastPath = false, store, }: { @@ -58,8 +59,25 @@ export class EnqueueSystem { workerId?: string; runnerId?: string; skipRunLock?: boolean; - /** When true, include TTL in the queued message (only for first enqueue from trigger). Default false. */ + /** + * When true, arm the run's TTL on the queued message. Set by every path that is the run's + * first real entry into the queue: trigger, a delayed run coming due, and the pending-version + * promotion. Waitpoint and checkpoint re-enqueues must not re-arm it. Default false. + */ includeTtl?: boolean; + /** + * When true, the scheduling-delay clock starts at the run's queue position (its + * `queueTimestamp`, which for a delayed run is `delayUntil`) rather than now. Set only where + * the run was genuinely eligible to execute from that moment: trigger and a delayed run + * coming due. + * + * Deliberately separate from `includeTtl`, because the two disagree on the pending-version + * promotion: that promotion is the run's first entry into the queue (so TTL arms there), but + * the run was NOT runnable while it waited for a worker version, and its `queueTimestamp` + * still holds the original trigger time. Anchoring there would bill the whole wait-for-deploy + * period as queue scheduling delay. Default false, so a new call site has to opt in. + */ + anchorEligibilityAtQueuePosition?: boolean; /** When true, allow the queue to push directly to worker queue if concurrency is available. */ enableFastPath?: boolean; /** @@ -98,10 +116,10 @@ export class EnqueueSystem { // Force development runs to use the environment id as the worker queue. const workerQueue = env.type === "DEVELOPMENT" ? env.id : run.workerQueue; - const timestamp = (run.queueTimestamp ?? run.createdAt).getTime() - run.priorityMs; + const queuePositionMs = (run.queueTimestamp ?? run.createdAt).getTime(); + const timestamp = queuePositionMs - run.priorityMs; + const eligibleAtMs = anchorEligibilityAtQueuePosition ? queuePositionMs : Date.now(); - // Include TTL only when explicitly requested (first enqueue from trigger). - // Re-enqueues (waitpoint, checkpoint, delayed, pending version) must not add TTL. let ttlExpiresAt: number | undefined; if (includeTtl && run.ttl) { const expireAt = parseNaturalLanguageDuration(run.ttl); @@ -124,6 +142,7 @@ export class EnqueueSystem { queue: run.queue, concurrencyKey: run.concurrencyKey ?? undefined, timestamp, + eligibleAtMs, attempt: 0, ttlExpiresAt, }, diff --git a/internal-packages/run-engine/src/engine/tests/pendingVersion.test.ts b/internal-packages/run-engine/src/engine/tests/pendingVersion.test.ts index b13438aa26c..9351b60e99f 100644 --- a/internal-packages/run-engine/src/engine/tests/pendingVersion.test.ts +++ b/internal-packages/run-engine/src/engine/tests/pendingVersion.test.ts @@ -314,7 +314,7 @@ describe("RunEngine pending version", () => { ); containerTest( - "PENDING_VERSION re-enqueue arms TTL on the queued message", + "PENDING_VERSION re-enqueue arms TTL but anchors the scheduling-delay clock at the promotion", async ({ prisma, redisOptions }) => { // When a run enters PENDING_VERSION (background worker doesn't yet have // the task), the first enqueue happens but the message is dequeued and @@ -407,6 +407,7 @@ describe("RunEngine pending version", () => { // Now a worker arrives WITH the task — pendingVersionSystem // re-enqueues the run. + const beforePromotion = Date.now(); await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); await setTimeout(1000); @@ -424,6 +425,21 @@ describe("RunEngine pending version", () => { assertNonNullable(message); expect(message.ttlExpiresAt).toBeDefined(); expect(typeof message.ttlExpiresAt).toBe("number"); + + const queuePositionMs = (run.queueTimestamp ?? run.createdAt).getTime(); + + expect( + message.eligibleAtMs, + "scheduling-delay clock must start at the promotion, since the run was not runnable while it waited for a worker version" + ).toBeGreaterThanOrEqual(beforePromotion); + expect( + message.eligibleAtMs, + "anchoring at the original queue position would bill the whole wait-for-deploy period as queue scheduling delay" + ).toBeGreaterThan(queuePositionMs); + expect( + message.timestamp, + "ordering keeps the original queue position, so waiting for a deployment must not cost the run its place in line" + ).toBe(queuePositionMs - run.priorityMs); } finally { await engine.quit(); } diff --git a/internal-packages/run-engine/src/engine/tests/ttl.test.ts b/internal-packages/run-engine/src/engine/tests/ttl.test.ts index 949e47f8574..e33b361abdb 100644 --- a/internal-packages/run-engine/src/engine/tests/ttl.test.ts +++ b/internal-packages/run-engine/src/engine/tests/ttl.test.ts @@ -293,7 +293,12 @@ describe("RunEngine ttl", () => { ); assertNonNullable(messageAfterTrigger); expect(messageAfterTrigger.ttlExpiresAt).toBeDefined(); + // First enqueue anchors the scheduling-delay clock at the trigger time. + expect(messageAfterTrigger.eligibleAtMs).toBe( + (run.queueTimestamp ?? run.createdAt).getTime() + ); + const beforeReenqueue = Date.now(); await engine.enqueueSystem.enqueueRun({ run, env: authenticatedEnvironment, @@ -308,6 +313,10 @@ describe("RunEngine ttl", () => { ); assertNonNullable(messageAfterReenqueue); expect(messageAfterReenqueue.ttlExpiresAt).toBeUndefined(); + // Re-enqueues anchor to now so the wait metric measures only this queue stint, + // while the ordering timestamp keeps the run's original position. + expect(messageAfterReenqueue.eligibleAtMs).toBeGreaterThanOrEqual(beforeReenqueue); + expect(messageAfterReenqueue.timestamp).toBe(messageAfterTrigger.timestamp); } finally { await engine.quit(); } diff --git a/internal-packages/run-engine/src/engine/types.ts b/internal-packages/run-engine/src/engine/types.ts index bb1d6eb2fa9..f37ec7df50a 100644 --- a/internal-packages/run-engine/src/engine/types.ts +++ b/internal-packages/run-engine/src/engine/types.ts @@ -16,6 +16,7 @@ import { } from "@trigger.dev/redis-worker"; import type { ControlPlaneResolver } from "./controlPlaneResolver.js"; import type { FairQueueSelectionStrategyOptions } from "../run-queue/fairQueueSelectionStrategy.js"; +import type { RunQueueMetricsEmitter } from "../run-queue/index.js"; import type { MinimalAuthenticatedEnvironment } from "../shared/index.js"; import type { LockRetryConfig } from "./locking.js"; import type { workerCatalog } from "./workerCatalog.js"; @@ -90,6 +91,8 @@ export type RunEngineOptions = { defaultEnvConcurrency?: number; defaultEnvConcurrencyBurstFactor?: number; logLevel?: LogLevel; + /** Optional queue-metrics emitter; enables gauge + counter emission from the RunQueue. */ + queueMetrics?: RunQueueMetricsEmitter; queueSelectionStrategyOptions?: Pick< FairQueueSelectionStrategyOptions, "parentQueueLimit" | "tracer" | "biases" | "reuseSnapshotCount" | "maximumEnvCount" diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index a0571206538..58225cc5051 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -5,6 +5,7 @@ import { type RedisOptions, type Result, } from "@internal/redis"; +import { createMetricsGaugeComputeLua } from "@internal/metrics-pipeline"; import type { Attributes, Meter, @@ -57,6 +58,99 @@ const SemanticAttributes = { ORG_ID: "runqueue.orgId", }; +// Prelude spliced at the top of every gauge-carrying script: declares the gauge slot and +// the return wrapper. A splice fills __qm_g; every return goes through __qmret so the reply +// is always {original, gauge}. A nil original becomes false, else Lua drops it from the +// multi-bulk reply (which would swallow the gauge on the dequeue throttle paths). +const QUEUE_METRICS_GAUGE_PRELUDE = ` +local __qm_g = false +local function __qmret(r) if r == nil then r = false end return {r, __qm_g} end`; + +// Fresh-read gauge for splice points with no reusable locals: enqueue slow-path (before +// return 0) and the base dequeue top. Gated on the last ARGV so it is inert unless the +// caller opts in. CK queues emit per-subqueue depth (queue_name aggregates via the MV). +const QUEUE_METRICS_GAUGE_LUA = createMetricsGaugeComputeLua({ + enabledArg: "ARGV[#ARGV] == '1'", + queued: "redis.call('ZCARD', queueKey)", + running: "redis.call('SCARD', queueCurrentConcurrencyKey)", + queueLimit: "redis.call('GET', queueConcurrencyLimitKey) or '1000000'", + envQueued: "redis.call('ZCARD', envQueueKey)", + envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)", + envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", +}); + +// Enqueue fast-path gauge: the admission check already computed queueCurrent/envCurrent/ +// queueLimit/envLimit, so reuse them (only 2 ZCARDs stay fresh). Fast path was taken, so +// cc < lim and thr is always 0 — reusing the effective queueLimit is fine (max() recovers raw). +const QUEUE_METRICS_ENQUEUE_FASTPATH_GAUGE_LUA = createMetricsGaugeComputeLua({ + enabledArg: "ARGV[#ARGV] == '1'", + queued: "redis.call('ZCARD', queueKey)", + running: "queueCurrent", + queueLimit: "queueLimit", + envQueued: "redis.call('ZCARD', envQueueKey)", + envRunning: "envCurrent", + envLimit: "envLimit", +}); + +// CK-health extras: distinct backlogged keys + most-starved head-of-line wait (ckIndex scores +// are per-subqueue oldest timestamps). Needs ckIndexKey/currentTime locals; clamps future scores. +const QUEUE_METRICS_CK_GAUGE_EXTRAS = { + preamble: `local __ckhead = redis.call('ZRANGE', ckIndexKey, 0, 0, 'WITHSCORES') + local __ckwait = 0 + if #__ckhead > 0 then __ckwait = math.floor(math.max(0, (tonumber(currentTime) or 0) - (tonumber(__ckhead[2]) or 0))) end`, + ckBacklogged: "redis.call('ZCARD', ckIndexKey)", + ckMaxWaitMs: "__ckwait", +}; + +// CK enqueue variants of the two gauges above, extended with the CK-health tail. +const QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ + enabledArg: "ARGV[#ARGV] == '1'", + queued: "redis.call('ZCARD', queueKey)", + running: "redis.call('SCARD', queueCurrentConcurrencyKey)", + queueLimit: "redis.call('GET', queueConcurrencyLimitKey) or '1000000'", + envQueued: "redis.call('ZCARD', envQueueKey)", + envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)", + envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", + ...QUEUE_METRICS_CK_GAUGE_EXTRAS, +}); + +const QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA = createMetricsGaugeComputeLua({ + enabledArg: "ARGV[#ARGV] == '1'", + queued: "redis.call('ZCARD', queueKey)", + running: "queueCurrent", + queueLimit: "queueLimit", + envQueued: "redis.call('ZCARD', envQueueKey)", + envRunning: "envCurrent", + envLimit: "envLimit", + ...QUEUE_METRICS_CK_GAUGE_EXTRAS, +}); + +// CK dequeue: depth/running from the per-base-queue aggregate counters the run-queue already +// maintains (two O(1) GETs, not a per-variant scan). thr suppressed — an aggregate cc >= per-CK +// limit would over-report; per-CK throttle is caught by the per-subqueue enqueue gauges. +const QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ + enabledArg: "ARGV[#ARGV] == '1'", + queued: "redis.call('GET', lengthCounterKey) or '0'", + running: "redis.call('GET', runningCounterKey) or '0'", + queueLimit: "redis.call('GET', queueConcurrencyLimitKey) or '1000000'", + envQueued: "redis.call('ZCARD', envQueueKey)", + envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)", + envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", + throttledExpr: "false", + ...QUEUE_METRICS_CK_GAUGE_EXTRAS, +}); + +/** Injected queue-metrics stream emitter; all calls are no-ops when metrics are disabled. */ +export interface RunQueueMetricsEmitter { + enabledSync(): boolean; + /** enabled AND sampled-in; gates high-frequency sampled emissions (the Lua gauge). */ + sampledSync(): boolean; + /** Counter event (cumulative odometer). */ + emit(shardKey: string, fields: Record<string, string | number>): void; + /** Gauge snapshot read inside the queue-op Lua and returned on the reply. */ + emitGauge(shardKey: string, fields: Record<string, string | number>): void; +} + export type RunQueueOptions = { name: string; tracer: Tracer; @@ -93,6 +187,8 @@ export type RunQueueOptions = { disabled?: boolean; }; meter?: Meter; + /** When set, enqueue/dequeue/ack/nack/dlq emit queue-metrics events (gated on the emitter's flag). */ + queueMetrics?: RunQueueMetricsEmitter; dequeueBlockingTimeoutSeconds?: number; concurrencySweeper?: { scanSchedule?: string; @@ -458,6 +554,109 @@ export class RunQueue { ); } + /** + * Live per-concurrency-key breakdown of a queue's backlog, most-starved first. + * Reads the ckIndex zset (members = CK subqueue names, scores = oldest-message + * timestamps), so only keys with queued work appear; running-only keys do not. + */ + public async concurrencyKeyBreakdown( + env: MinimalAuthenticatedEnvironment, + queue: string, + options?: { limit?: number } + ): Promise<{ + totalBackloggedKeys: number; + keys: Array<{ + concurrencyKey: string; + queued: number; + running: number; + oldestEnqueuedAt: number; + }>; + }> { + const limit = options?.limit ?? 50; + const ckIndexKey = this.keys.ckIndexKeyFromQueue(this.keys.queueKey(env, queue)); + + const indexPipeline = this.redis.pipeline(); + indexPipeline.zcard(ckIndexKey); + indexPipeline.zrange(ckIndexKey, 0, limit - 1, "WITHSCORES"); + const indexResults = await indexPipeline.exec(); + if (!indexResults) return { totalBackloggedKeys: 0, keys: [] }; + + const [totalErr, totalVal] = indexResults[0]; + const [rangeErr, rangeVal] = indexResults[1]; + const totalBackloggedKeys = totalErr || totalVal == null ? 0 : (totalVal as number); + const flat = rangeErr || rangeVal == null ? [] : (rangeVal as string[]); + + const members: Array<{ member: string; score: number }> = []; + for (let i = 0; i < flat.length; i += 2) { + members.push({ member: flat[i], score: Number(flat[i + 1]) }); + } + if (members.length === 0) return { totalBackloggedKeys, keys: [] }; + + const statsPipeline = this.redis.pipeline(); + for (const { member } of members) { + statsPipeline.zcard(member); + statsPipeline.scard(this.keys.queueCurrentConcurrencyKeyFromQueue(member)); + } + const stats = await statsPipeline.exec(); + + const keys = members.map(({ member, score }, i) => { + const queuedResult = stats?.[i * 2]; + const runningResult = stats?.[i * 2 + 1]; + return { + concurrencyKey: this.#concurrencyKeyFromQueue(member) ?? "", + queued: queuedResult && !queuedResult[0] ? ((queuedResult[1] as number) ?? 0) : 0, + running: runningResult && !runningResult[0] ? ((runningResult[1] as number) ?? 0) : 0, + oldestEnqueuedAt: score, + }; + }); + + return { totalBackloggedKeys, keys }; + } + + /** + * Live "now" stats for a specific set of concurrency keys — the current page of the paginated + * per-key table. Unlike concurrencyKeyBreakdown (which reads the top of the ckIndex), this + * targets exactly the given keys, so the table can enrich its ClickHouse-ranked page without + * scanning the whole index: O(keys) via one pipeline, independent of total key cardinality. + * Keys with no live backlog come back as zeros with a null oldest-enqueue time. + */ + public async concurrencyKeyLiveStats( + env: MinimalAuthenticatedEnvironment, + queue: string, + concurrencyKeys: string[] + ): Promise<Map<string, { queued: number; running: number; oldestEnqueuedAt: number | null }>> { + const result = new Map< + string, + { queued: number; running: number; oldestEnqueuedAt: number | null } + >(); + if (concurrencyKeys.length === 0) return result; + + const ckIndexKey = this.keys.ckIndexKeyFromQueue(this.keys.queueKey(env, queue)); + + const pipeline = this.redis.pipeline(); + for (const concurrencyKey of concurrencyKeys) { + const member = this.keys.queueKey(env, queue, concurrencyKey); + pipeline.zcard(member); // queued in this key's subqueue + pipeline.scard(this.keys.queueCurrentConcurrencyKeyFromQueue(member)); // running + pipeline.zscore(ckIndexKey, member); // oldest-enqueued score (null once the key drains) + } + const res = await pipeline.exec(); + if (!res) return result; + + concurrencyKeys.forEach((concurrencyKey, i) => { + const queuedResult = res[i * 3]; + const runningResult = res[i * 3 + 1]; + const scoreResult = res[i * 3 + 2]; + const queued = queuedResult && !queuedResult[0] ? ((queuedResult[1] as number) ?? 0) : 0; + const running = runningResult && !runningResult[0] ? ((runningResult[1] as number) ?? 0) : 0; + const rawScore = scoreResult && !scoreResult[0] ? scoreResult[1] : null; + const oldestEnqueuedAt = rawScore != null ? Number(rawScore) : null; + result.set(concurrencyKey, { queued, running, oldestEnqueuedAt }); + }); + + return result; + } + public async lengthOfEnvQueue(env: MinimalAuthenticatedEnvironment) { return this.redis.zcard(this.keys.envQueueKey(env)); } @@ -751,6 +950,8 @@ export class RunQueue { span.setAttribute("fastPath", fastPathTaken); + this.#emitQueueMetric(queueKey, { op: "enqueue", q: queueKey }); + if (!fastPathTaken && !skipDequeueProcessing) { // Slow path: schedule the dequeue job to move the message from queue to worker queue await this.worker.enqueueOnce({ @@ -810,6 +1011,15 @@ export class RunQueue { ...flattenAttributes(dequeuedMessage.message, "message"), }); + const startedFields: Record<string, string | number> = { + op: "started", + q: dequeuedMessage.message.queue, + }; + if (typeof dequeuedMessage.message.eligibleAtMs === "number") { + startedFields.wait = Math.max(0, Date.now() - dequeuedMessage.message.eligibleAtMs); + } + this.#emitQueueMetric(dequeuedMessage.message.queue, startedFields); + return dequeuedMessage; }, { @@ -877,6 +1087,8 @@ export class RunQueue { message, removeFromWorkerQueue: options?.removeFromWorkerQueue, }); + + this.#emitQueueMetric(message.queue, { op: "ack", q: message.queue }); }, { kind: SpanKind.CONSUMER, @@ -934,6 +1146,7 @@ export class RunQueue { message.attempt = message.attempt + 1; if (message.attempt >= maxAttempts) { await this.#callMoveToDeadLetterQueue({ message }); + this.#emitQueueMetric(message.queue, { op: "dlq", q: message.queue }); return false; } } @@ -960,6 +1173,8 @@ export class RunQueue { await this.#callNackMessage({ message, retryAt }); + this.#emitQueueMetric(message.queue, { op: "nack", q: message.queue }); + return true; }, { @@ -1831,6 +2046,57 @@ export class RunQueue { * * @returns true if the fast path was taken (message pushed directly to worker queue) */ + #queueMetricsGaugeArg(): string { + // Gauge gate ARGV: enabled AND sampled-in (sampling applies to the gauge, not counters). + return this.options.queueMetrics?.sampledSync() ? "1" : "0"; + } + + // Gauge returned on a script reply as a flat [ql, cc, lim, eql, ec, elim, thr] array, + // plus an optional [ckq, ckw] tail on CK-path scripts. + // Unlike counters, gauges are NOT base-normalized: the q label keeps its :ck: suffix so + // the CK-aggregate and per-subqueue readings stay distinguishable; the consumer's mapEntry + // strips :ck: to the base queue_name and the MV maxes them into one row. + #emitGauge(queue: string, gauge: number[]): void { + if (!Array.isArray(gauge) || gauge.length < 7) return; + const [ql, cc, lim, eql, ec, elim, thr, ckq, ckw] = gauge; + const fields: Record<string, string | number> = { + op: "gauge", + q: queue, + ql, + cc, + lim, + eql, + ec, + elim, + thr, + }; + if (gauge.length >= 9) { + fields.ckq = ckq; + fields.ckw = ckw; + } + this.options.queueMetrics?.emitGauge(queue, fields); + } + + #concurrencyKeyFromQueue(queue: string): string | undefined { + const idx = queue.indexOf(":ck:"); + return idx === -1 || idx + 4 >= queue.length ? undefined : queue.slice(idx + 4); + } + + #emitQueueMetric(shardKey: string, fields: Record<string, string | number>): void { + // Counters roll up per BASE queue: normalize the CK-qualified queue to its base so all + // concurrency keys share one monotonic odometer (and one shard/order key), matching the + // base queue_name the consumer buckets on. A real concurrency key rides along as `ck`, + // driving a SEPARATE per-key odometer on the same entry (per-key history tier). + const baseQueue = this.keys.baseQueueKeyFromQueue(shardKey); + let baseFields = fields; + if (typeof fields.q === "string") { + baseFields = { ...fields, q: this.keys.baseQueueKeyFromQueue(fields.q) }; + const ck = this.#concurrencyKeyFromQueue(fields.q); + if (ck && ck !== "*") baseFields.ck = ck; + } + this.options.queueMetrics?.emit(baseQueue, baseFields); + } + async #callEnqueueMessage( message: OutputPayloadV2, ttlInfo?: { @@ -1869,6 +2135,7 @@ export class RunQueue { const messageScore = String(message.timestamp); const currentTime = String(Date.now()); const enableFastPathArg = enableFastPath ? "1" : "0"; + const metricsGaugeArg = this.#queueMetricsGaugeArg(); const defaultEnvConcurrencyLimit = String(this.options.defaultEnvConcurrency); const defaultEnvConcurrencyBurstFactor = String( this.options.defaultEnvConcurrencyBurstFactor ?? 1.0 @@ -1892,7 +2159,8 @@ export class RunQueue { service: this.name, }); - let result: number; + // Every gauge-carrying script returns a 2-tuple [originalReturn, gauge|null]. + let result: [number, number[] | null]; // Use CK-aware enqueue for messages with concurrency keys if (message.concurrencyKey) { @@ -1935,7 +2203,8 @@ export class RunQueue { currentTime, enableFastPathArg, ckKeyPrefix, - String(this.counterTtlSeconds) + String(this.counterTtlSeconds), + metricsGaugeArg ); } else { result = await this.redis.enqueueMessageCkTracked( @@ -1967,7 +2236,8 @@ export class RunQueue { currentTime, enableFastPathArg, ckKeyPrefix, - String(this.counterTtlSeconds) + String(this.counterTtlSeconds), + metricsGaugeArg ); } } else if (ttlInfo) { @@ -1998,7 +2268,8 @@ export class RunQueue { defaultEnvConcurrencyLimit, defaultEnvConcurrencyBurstFactor, currentTime, - enableFastPathArg + enableFastPathArg, + metricsGaugeArg ); } else { result = await this.redis.enqueueMessage( @@ -2024,11 +2295,14 @@ export class RunQueue { defaultEnvConcurrencyLimit, defaultEnvConcurrencyBurstFactor, currentTime, - enableFastPathArg + enableFastPathArg, + metricsGaugeArg ); } - return result === 1; + const [enqueueResult, gauge] = result; + if (gauge) this.#emitGauge(queueName, gauge); + return enqueueResult === 1; } async #callDequeueMessagesFromQueue({ @@ -2081,7 +2355,9 @@ export class RunQueue { maxCount, }); - const result = await this.redis.dequeueMessagesFromQueue( + const metricsGaugeArg = this.#queueMetricsGaugeArg(); + + const reply = await this.redis.dequeueMessagesFromQueue( //keys messageQueue, queueConcurrencyLimitKey, @@ -2099,9 +2375,16 @@ export class RunQueue { String(this.options.defaultEnvConcurrency), String(this.options.defaultEnvConcurrencyBurstFactor ?? 1), this.options.redis.keyPrefix ?? "", - String(maxCount) + String(maxCount), + metricsGaugeArg ); + // Reply is [flatMessages|null, gauge|null]: emit the gauge (read atomically inside + // the script, present on the throttle/empty paths too) and keep element 0 as the array. + const gauge = reply?.[1] ?? null; + if (gauge) this.#emitGauge(messageQueue, gauge); + const result = reply?.[0] ?? null; + if (!result) { span.setAttribute("message_count", 0); @@ -2202,8 +2485,11 @@ export class RunQueue { }); const lengthCounterKey = this.keys.queueLengthCounterKeyFromQueue(ckWildcardQueue); + const runningCounterKey = this.keys.queueRunningCounterKeyFromQueue(ckWildcardQueue); + + const metricsGaugeArg = this.#queueMetricsGaugeArg(); - const result = await this.redis.dequeueMessagesFromCkQueueTracked( + const reply = await this.redis.dequeueMessagesFromCkQueueTracked( //keys ckIndexKey, queueConcurrencyLimitKey, @@ -2215,15 +2501,22 @@ export class RunQueue { masterQueueKey, ttlQueueKey, lengthCounterKey, + runningCounterKey, //args ckWildcardQueue, String(Date.now()), String(this.options.defaultEnvConcurrency), String(this.options.defaultEnvConcurrencyBurstFactor ?? 1), this.options.redis.keyPrefix ?? "", - String(maxCount) + String(maxCount), + metricsGaugeArg ); + // Reply is [flatMessages|null, gauge|null]; the CK aggregate gauge rides here. + const gauge = reply?.[1] ?? null; + if (gauge) this.#emitGauge(ckWildcardQueue, gauge); + const result = reply?.[0] ?? null; + if (!result) { span.setAttribute("message_count", 0); return []; @@ -3062,6 +3355,8 @@ local defaultEnvConcurrencyBurstFactor = ARGV[7] local currentTime = ARGV[8] local enableFastPath = ARGV[9] +${QUEUE_METRICS_GAUGE_PRELUDE} + -- Fast path: check if we can skip the queue and go directly to worker queue if enableFastPath == '1' then local available = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'LIMIT', 0, 1) @@ -3083,7 +3378,8 @@ if enableFastPath == '1' then redis.call('SADD', queueCurrentConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) redis.call('RPUSH', workerQueueKey, messageKeyValue) - return 1 +${QUEUE_METRICS_ENQUEUE_FASTPATH_GAUGE_LUA} + return __qmret(1) end end end @@ -3113,8 +3409,9 @@ redis.call('SREM', queueCurrentConcurrencyKey, messageId) redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) +${QUEUE_METRICS_GAUGE_LUA} -return 0 +return __qmret(0) `, }); @@ -3153,6 +3450,8 @@ local defaultEnvConcurrencyBurstFactor = ARGV[9] local currentTime = ARGV[10] local enableFastPath = ARGV[11] +${QUEUE_METRICS_GAUGE_PRELUDE} + -- Fast path: check if we can skip the queue and go directly to worker queue if enableFastPath == '1' then local available = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'LIMIT', 0, 1) @@ -3174,8 +3473,9 @@ if enableFastPath == '1' then redis.call('SADD', queueCurrentConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) redis.call('RPUSH', workerQueueKey, messageKeyValue) +${QUEUE_METRICS_ENQUEUE_FASTPATH_GAUGE_LUA} -- Skip TTL sorted set: the expireRun worker job handles TTL expiry independently - return 1 + return __qmret(1) end end end @@ -3208,8 +3508,9 @@ redis.call('SREM', queueCurrentConcurrencyKey, messageId) redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) +${QUEUE_METRICS_GAUGE_LUA} -return 0 +return __qmret(0) `, }); @@ -3246,6 +3547,8 @@ local defaultEnvConcurrencyBurstFactor = ARGV[8] local currentTime = ARGV[9] local enableFastPath = ARGV[10] +${QUEUE_METRICS_GAUGE_PRELUDE} + -- Fast path: check if we can skip the queue and go directly to worker queue if enableFastPath == '1' then local available = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'LIMIT', 0, 1) @@ -3268,7 +3571,8 @@ if enableFastPath == '1' then redis.call('SADD', queueCurrentConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) redis.call('RPUSH', workerQueueKey, messageKeyValue) - return 1 +${QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA} + return __qmret(1) end end end @@ -3304,8 +3608,9 @@ redis.call('SREM', queueCurrentConcurrencyKey, messageId) redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) +${QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA} -return 0 +return __qmret(0) `, }); @@ -3344,6 +3649,8 @@ local defaultEnvConcurrencyBurstFactor = ARGV[10] local currentTime = ARGV[11] local enableFastPath = ARGV[12] +${QUEUE_METRICS_GAUGE_PRELUDE} + -- Fast path: check if we can skip the queue and go directly to worker queue if enableFastPath == '1' then local available = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'LIMIT', 0, 1) @@ -3365,8 +3672,9 @@ if enableFastPath == '1' then redis.call('SADD', queueCurrentConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) redis.call('RPUSH', workerQueueKey, messageKeyValue) +${QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA} -- Skip TTL sorted set: the expireRun worker job handles TTL expiry independently - return 1 + return __qmret(1) end end end @@ -3405,8 +3713,9 @@ redis.call('SREM', queueCurrentConcurrencyKey, messageId) redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) +${QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA} -return 0 +return __qmret(0) `, }); @@ -3455,6 +3764,8 @@ local keyPrefix = ARGV[11] -- TTL (seconds) applied to counter lazy-init SETs local counterTtl = ARGV[12] +${QUEUE_METRICS_GAUGE_PRELUDE} + -- Fast path: check if we can skip the queue and go directly to worker queue if enableFastPath == '1' then local available = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'LIMIT', 0, 1) @@ -3476,10 +3787,11 @@ if enableFastPath == '1' then redis.call('SADD', queueCurrentConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) redis.call('RPUSH', workerQueueKey, messageKeyValue) +${QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA} -- Fast-path skips the CK variant zset entirely; lengthCounter is unchanged. -- runningCounter is bumped later by dequeueMessageFromKeyTracked when the -- worker pulls the message from the worker queue. - return 1 + return __qmret(1) end end end @@ -3531,8 +3843,9 @@ redis.call('SREM', queueCurrentConcurrencyKey, messageId) redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) +${QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA} -return 0 +return __qmret(0) `, }); @@ -3576,6 +3889,8 @@ local keyPrefix = ARGV[13] -- TTL (seconds) applied to counter lazy-init SETs local counterTtl = ARGV[14] +${QUEUE_METRICS_GAUGE_PRELUDE} + -- Fast path: check if we can skip the queue and go directly to worker queue if enableFastPath == '1' then local available = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'LIMIT', 0, 1) @@ -3597,7 +3912,8 @@ if enableFastPath == '1' then redis.call('SADD', queueCurrentConcurrencyKey, messageId) redis.call('SADD', envCurrentConcurrencyKey, messageId) redis.call('RPUSH', workerQueueKey, messageKeyValue) - return 1 +${QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA} + return __qmret(1) end end end @@ -3645,8 +3961,9 @@ redis.call('SREM', queueCurrentConcurrencyKey, messageId) redis.call('SREM', envCurrentConcurrencyKey, messageId) redis.call('SREM', queueCurrentDequeuedKey, messageId) redis.call('SREM', envCurrentDequeuedKey, messageId) +${QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA} -return 0 +return __qmret(0) `, }); @@ -3891,6 +4208,8 @@ local defaultEnvConcurrencyLimit = ARGV[3] local defaultEnvConcurrencyBurstFactor = ARGV[4] local keyPrefix = ARGV[5] local maxCount = tonumber(ARGV[6] or '1') +${QUEUE_METRICS_GAUGE_PRELUDE} +${QUEUE_METRICS_GAUGE_LUA} -- Check current env concurrency against the limit local envCurrentConcurrency = tonumber(redis.call('SCARD', envCurrentConcurrencyKey) or '0') @@ -3899,7 +4218,7 @@ local envConcurrencyLimitBurstFactor = tonumber(redis.call('GET', envConcurrency local envConcurrencyLimitWithBurstFactor = math.floor(envConcurrencyLimit * envConcurrencyLimitBurstFactor) if envCurrentConcurrency >= envConcurrencyLimitWithBurstFactor then - return nil + return __qmret(nil) end -- Check current queue concurrency against the limit @@ -3909,7 +4228,7 @@ local totalQueueConcurrencyLimit = queueConcurrencyLimit -- Check condition only if concurrencyLimit exists if queueCurrentConcurrency >= totalQueueConcurrencyLimit then - return nil + return __qmret(nil) end -- Calculate how many messages we can actually dequeue based on concurrency limits @@ -3918,14 +4237,14 @@ local queueAvailableCapacity = totalQueueConcurrencyLimit - queueCurrentConcurre local actualMaxCount = math.min(maxCount, envAvailableCapacity, queueAvailableCapacity) if actualMaxCount <= 0 then - return nil + return __qmret(nil) end -- Attempt to dequeue messages up to actualMaxCount local messages = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'WITHSCORES', 'LIMIT', 0, actualMaxCount) if #messages == 0 then - return nil + return __qmret(nil) end local results = {} @@ -3991,7 +4310,7 @@ else end -- Return results as a flat array: [messageId1, messageScore1, messagePayload1, messageId2, messageScore2, messagePayload2, ...] -return results +return __qmret(results) `, }); @@ -4145,7 +4464,7 @@ return results // (normal dequeue, TTL-expired, or stale-orphan path — all of which were // counted at enqueue time). this.redis.defineCommand("dequeueMessagesFromCkQueueTracked", { - numberOfKeys: 10, + numberOfKeys: 11, lua: ` local ckIndexKey = KEYS[1] local queueConcurrencyLimitKey = KEYS[2] @@ -4157,6 +4476,7 @@ local envQueueKey = KEYS[7] local masterQueueKey = KEYS[8] local ttlQueueKey = KEYS[9] local lengthCounterKey = KEYS[10] +local runningCounterKey = KEYS[11] local ckWildcardName = ARGV[1] local currentTime = tonumber(ARGV[2]) @@ -4164,6 +4484,8 @@ local defaultEnvConcurrencyLimit = ARGV[3] local defaultEnvConcurrencyBurstFactor = ARGV[4] local keyPrefix = ARGV[5] local maxCount = tonumber(ARGV[6] or '1') +${QUEUE_METRICS_GAUGE_PRELUDE} +${QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA} local function decrLengthCounter() if tonumber(redis.call('GET', lengthCounterKey) or '0') > 0 then @@ -4178,7 +4500,7 @@ local envConcurrencyLimitBurstFactor = tonumber(redis.call('GET', envConcurrency local envConcurrencyLimitWithBurstFactor = math.floor(envConcurrencyLimit * envConcurrencyLimitBurstFactor) if envCurrentConcurrency >= envConcurrencyLimitWithBurstFactor then - return nil + return __qmret(nil) end local queueConcurrencyLimit = math.min(tonumber(redis.call('GET', queueConcurrencyLimitKey) or '1000000'), envConcurrencyLimit) @@ -4187,7 +4509,7 @@ local envAvailableCapacity = envConcurrencyLimitWithBurstFactor - envCurrentConc local actualMaxCount = math.min(maxCount, envAvailableCapacity) if actualMaxCount <= 0 then - return nil + return __qmret(nil) end local ckQueues = redis.call('ZRANGEBYSCORE', ckIndexKey, '-inf', tostring(currentTime), 'LIMIT', 0, actualMaxCount * 3) @@ -4199,7 +4521,7 @@ if #ckQueues == 0 then else redis.call('ZADD', masterQueueKey, anyIdx[2], ckWildcardName) end - return nil + return __qmret(nil) end local results = {} @@ -4281,7 +4603,7 @@ else redis.call('ZADD', masterQueueKey, earliestIdx[2], ckWildcardName) end -return results +return __qmret(results) `, }); @@ -5199,8 +5521,9 @@ declare module "@internal/redis" { defaultEnvConcurrencyBurstFactor: string, currentTime: string, enableFastPath: string, - callback?: Callback<number> - ): Result<number, Context>; + metricsEnabled: string, + callback?: Callback<[number, number[] | null]> + ): Result<[number, number[] | null], Context>; enqueueMessageWithTtl( //keys @@ -5229,8 +5552,9 @@ declare module "@internal/redis" { defaultEnvConcurrencyBurstFactor: string, currentTime: string, enableFastPath: string, - callback?: Callback<number> - ): Result<number, Context>; + metricsEnabled: string, + callback?: Callback<[number, number[] | null]> + ): Result<[number, number[] | null], Context>; expireTtlRuns( //keys @@ -5265,8 +5589,9 @@ declare module "@internal/redis" { defaultEnvConcurrencyBurstFactor: string, keyPrefix: string, maxCount: string, - callback?: Callback<string[]> - ): Result<string[], Context>; + metricsEnabled: string, + callback?: Callback<[string[] | null, number[] | null]> + ): Result<[string[] | null, number[] | null], Context>; dequeueMessageFromWorkerQueueNonBlocking( workerQueueKey: string, @@ -5405,8 +5730,9 @@ declare module "@internal/redis" { defaultEnvConcurrencyBurstFactor: string, currentTime: string, enableFastPath: string, - callback?: Callback<number> - ): Result<number, Context>; + metricsEnabled: string, + callback?: Callback<[number, number[] | null]> + ): Result<[number, number[] | null], Context>; enqueueMessageWithTtlCk( //keys @@ -5437,8 +5763,9 @@ declare module "@internal/redis" { defaultEnvConcurrencyBurstFactor: string, currentTime: string, enableFastPath: string, - callback?: Callback<number> - ): Result<number, Context>; + metricsEnabled: string, + callback?: Callback<[number, number[] | null]> + ): Result<[number, number[] | null], Context>; dequeueMessagesFromCkQueue( //keys @@ -5551,8 +5878,9 @@ declare module "@internal/redis" { enableFastPath: string, keyPrefix: string, counterTtl: string, - callback?: Callback<number> - ): Result<number, Context>; + metricsEnabled: string, + callback?: Callback<[number, number[] | null]> + ): Result<[number, number[] | null], Context>; enqueueMessageWithTtlCkTracked( masterQueueKey: string, @@ -5585,8 +5913,9 @@ declare module "@internal/redis" { enableFastPath: string, keyPrefix: string, counterTtl: string, - callback?: Callback<number> - ): Result<number, Context>; + metricsEnabled: string, + callback?: Callback<[number, number[] | null]> + ): Result<[number, number[] | null], Context>; dequeueMessagesFromCkQueueTracked( ckIndexKey: string, @@ -5599,14 +5928,16 @@ declare module "@internal/redis" { masterQueueKey: string, ttlQueueKey: string, lengthCounterKey: string, + runningCounterKey: string, ckWildcardName: string, currentTime: string, defaultEnvConcurrencyLimit: string, defaultEnvConcurrencyBurstFactor: string, keyPrefix: string, maxCount: string, - callback?: Callback<string[]> - ): Result<string[], Context>; + metricsEnabled: string, + callback?: Callback<[string[] | null, number[] | null]> + ): Result<[string[] | null, number[] | null], Context>; dequeueMessageFromKeyTracked( messageKey: string, diff --git a/internal-packages/run-engine/src/run-queue/keyProducer.ts b/internal-packages/run-engine/src/run-queue/keyProducer.ts index 3be79a4fabc..0609b3d719b 100644 --- a/internal-packages/run-engine/src/run-queue/keyProducer.ts +++ b/internal-packages/run-engine/src/run-queue/keyProducer.ts @@ -141,8 +141,7 @@ export class RunQueueFullKeyProducer implements RunQueueKeyProducer { } queueConcurrencyLimitKeyFromQueue(queue: string) { - const concurrencyQueueName = queue.replace(/:ck:.+$/, ""); - return `${concurrencyQueueName}:${constants.CONCURRENCY_LIMIT_PART}`; + return `${this.baseQueueKeyFromQueue(queue)}:${constants.CONCURRENCY_LIMIT_PART}`; } queueCurrentConcurrencyKeyFromQueue(queue: string) { @@ -313,12 +312,14 @@ export class RunQueueFullKeyProducer implements RunQueueKeyProducer { } ckIndexKeyFromQueue(queue: string): string { - const baseQueue = queue.replace(/:ck:.+$/, ""); - return `${baseQueue}:${constants.CK_INDEX_PART}`; + return `${this.baseQueueKeyFromQueue(queue)}:${constants.CK_INDEX_PART}`; } + // indexOf instead of /:ck:.+$/ (queue names are user-controlled; polynomial regex). + // Only strips when at least one character follows ":ck:", matching the old semantics. baseQueueKeyFromQueue(queue: string): string { - return queue.replace(/:ck:.+$/, ""); + const idx = queue.indexOf(":ck:"); + return idx === -1 || idx + 4 >= queue.length ? queue : queue.slice(0, idx); } queueLengthCounterKey(env: RunQueueKeyProducerEnvironment, queue: string): string { @@ -342,7 +343,8 @@ export class RunQueueFullKeyProducer implements RunQueueKeyProducer { } toCkWildcard(queue: string): string { - return queue.replace(/:ck:.+$/, ":ck:*"); + const base = this.baseQueueKeyFromQueue(queue); + return base === queue ? queue : `${base}:ck:*`; } descriptorFromQueue(queue: string): QueueDescriptor { diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts new file mode 100644 index 00000000000..ebfc295470e --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -0,0 +1,397 @@ +import { createRedisClient } from "@internal/redis"; +import { redisTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { + allStreamKeys, + MetricsStreamEmitter, + type MetricDefinition, +} from "@internal/metrics-pipeline"; +import { Logger } from "@trigger.dev/core/logger"; +import { Decimal } from "@trigger.dev/database"; +import { setTimeout } from "node:timers/promises"; +import { describe, expect } from "vitest"; +import { FairQueueSelectionStrategy } from "./fairQueueSelectionStrategy.js"; +import { RunQueue } from "./index.js"; +import { RunQueueFullKeyProducer } from "./keyProducer.js"; +import type { InputPayload } from "./types.js"; + +const authenticatedEnvDev = { + id: "e1234", + type: "DEVELOPMENT" as const, + maximumConcurrencyLimit: 10, + concurrencyLimitBurstFactor: new Decimal(1.0), + project: { id: "p1234" }, + organization: { id: "o1234" }, +}; + +async function readAllEntries( + redisOptions: { + host: string; + port: number; + }, + definition: MetricDefinition +) { + const client = createRedisClient({ ...redisOptions, keyPrefix: undefined }); + const entries: Array<{ id: string; fields: Record<string, string> }> = []; + for (const key of allStreamKeys(definition)) { + const raw = (await client.xrange(key, "-", "+")) as Array<[string, string[]]>; + for (const [id, flat] of raw) { + const fields: Record<string, string> = {}; + for (let i = 0; i + 1 < flat.length; i += 2) fields[flat[i]!] = flat[i + 1]!; + entries.push({ id, fields }); + } + } + await client.quit(); + return entries; +} + +// Gauges now land via a fire-and-forget Node XADD after the script reply (not synchronously +// inside the Lua), so reads must poll until the expected entries appear. +async function waitForEntries( + redisOptions: { host: string; port: number }, + definition: MetricDefinition, + predicate: (entries: Array<{ id: string; fields: Record<string, string> }>) => boolean, + timeoutMs = 5000 +) { + const start = Date.now(); + let entries = await readAllEntries(redisOptions, definition); + while (!predicate(entries)) { + if (Date.now() - start > timeoutMs) return entries; + await setTimeout(50); + entries = await readAllEntries(redisOptions, definition); + } + return entries; +} + +describe("RunQueue queue-metrics emission", () => { + redisTest("emits gauge + enqueue/started/ack events when enabled", async ({ redisContainer }) => { + const redis = { + keyPrefix: "runqueue:test:", + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }; + const definition: MetricDefinition = { + name: `qm_test_${Date.now()}`, + shardCount: 2, + consumerGroup: "cg", + maxLen: 1000, + }; + const emitter = new MetricsStreamEmitter({ + redis, + definition, + flag: { enabled: () => true }, + }); + + const queue = new RunQueue({ + name: "rq", + tracer: trace.getTracer("rq"), + defaultEnvConcurrency: 25, + logger: new Logger("RunQueue", "error"), + keys: new RunQueueFullKeyProducer(), + queueSelectionStrategy: new FairQueueSelectionStrategy({ + redis, + keys: new RunQueueFullKeyProducer(), + }), + redis, + queueMetrics: emitter, + }); + + const message: InputPayload = { + runId: "r-metrics", + taskIdentifier: "task/my-task", + orgId: "o1234", + projectId: "p1234", + environmentId: authenticatedEnvDev.id, + environmentType: "DEVELOPMENT", + queue: "task/my-task", + timestamp: Date.now(), + eligibleAtMs: Date.now() - 500, + attempt: 0, + }; + + try { + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message, + workerQueue: authenticatedEnvDev.id, + }); + await setTimeout(1000); + const dequeued = await queue.dequeueMessageFromWorkerQueue("c1", authenticatedEnvDev.id); + expect(dequeued?.messageId).toBe(message.runId); + await queue.acknowledgeMessage(message.orgId, message.runId); + await setTimeout(100); + + const entries = await waitForEntries(redis, definition, (es) => { + const seen = es.map((e) => e.fields.op); + return ["enqueue", "gauge", "started", "ack"].every((o) => seen.includes(o)); + }); + const ops = entries.map((e) => e.fields.op); + expect(ops).toContain("enqueue"); + expect(ops).toContain("gauge"); + expect(ops).toContain("started"); + expect(ops).toContain("ack"); + + const gauge = entries.find((e) => e.fields.op === "gauge"); + assertGauge(gauge); + expect(gauge!.fields.q).toContain("task/my-task"); + for (const f of ["ql", "cc", "lim", "eql", "ec", "elim", "thr"]) { + expect(gauge!.fields[f]).toBeDefined(); + } + // Non-CK scripts keep the 7-field gauge (no CK-health tail). + expect(gauge!.fields.ckq).toBeUndefined(); + expect(gauge!.fields.ckw).toBeUndefined(); + + // The first counter emission also seeds a cum=0 baseline (no wait); the real reading + // carries wait. Pick the reading (cum > 0). + const started = entries.find((e) => e.fields.op === "started" && Number(e.fields.cum) > 0); + expect(started!.fields.wait).toBeDefined(); + expect(Number(started!.fields.wait)).toBeGreaterThanOrEqual(0); + expect(Number(started!.fields.cum)).toBeGreaterThan(0); + } finally { + await queue.quit(); + await emitter.close(); + } + }); + + redisTest( + "emits a fast-path gauge reusing the admission-check locals", + async ({ redisContainer }) => { + const redis = { + keyPrefix: "runqueue:test:", + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }; + const definition: MetricDefinition = { + name: `qm_fp_${Date.now()}`, + shardCount: 2, + consumerGroup: "cg", + maxLen: 1000, + }; + const emitter = new MetricsStreamEmitter({ + redis, + definition, + flag: { enabled: () => true }, + }); + const queue = new RunQueue({ + name: "rq", + tracer: trace.getTracer("rq"), + defaultEnvConcurrency: 25, + logger: new Logger("RunQueue", "error"), + keys: new RunQueueFullKeyProducer(), + queueSelectionStrategy: new FairQueueSelectionStrategy({ + redis, + keys: new RunQueueFullKeyProducer(), + }), + redis, + queueMetrics: emitter, + }); + + const message: InputPayload = { + runId: "r-fastpath", + taskIdentifier: "task/my-task", + orgId: "o1234", + projectId: "p1234", + environmentId: authenticatedEnvDev.id, + environmentType: "DEVELOPMENT", + queue: "task/my-task", + timestamp: Date.now(), + attempt: 0, + }; + + try { + // enableFastPath + empty queue + zero concurrency => the Lua takes the fast path, + // so the gauge runs the reuse snippet (queueCurrent/envCurrent/queueLimit/envLimit). + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message, + workerQueue: authenticatedEnvDev.id, + enableFastPath: true, + }); + const dequeued = await queue.dequeueMessageFromWorkerQueue("c1", authenticatedEnvDev.id); + expect(dequeued?.messageId).toBe(message.runId); + + const entries = await waitForEntries( + redis, + definition, + (es) => + es.some((e) => e.fields.op === "gauge") && es.some((e) => e.fields.op === "enqueue") + ); + const gauge = entries.find((e) => e.fields.op === "gauge"); + assertGauge(gauge); + for (const f of ["ql", "cc", "lim", "eql", "ec", "elim", "thr"]) { + expect(gauge!.fields[f]).toBeDefined(); + } + // Fast path was taken => capacity was available => not throttled. + expect(gauge!.fields.thr).toBe("0"); + expect(entries.some((e) => e.fields.op === "enqueue")).toBe(true); + } finally { + await queue.quit(); + await emitter.close(); + } + } + ); + + redisTest("emits an aggregate gauge for CK queues at dequeue", async ({ redisContainer }) => { + const redis = { + keyPrefix: "runqueue:test:", + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }; + const definition: MetricDefinition = { + name: `qm_ck_${Date.now()}`, + shardCount: 2, + consumerGroup: "cg", + maxLen: 1000, + }; + const emitter = new MetricsStreamEmitter({ redis, definition, flag: { enabled: () => true } }); + const queue = new RunQueue({ + name: "rq", + tracer: trace.getTracer("rq"), + defaultEnvConcurrency: 25, + logger: new Logger("RunQueue", "error"), + keys: new RunQueueFullKeyProducer(), + queueSelectionStrategy: new FairQueueSelectionStrategy({ + redis, + keys: new RunQueueFullKeyProducer(), + }), + redis, + queueMetrics: emitter, + }); + + const message: InputPayload = { + runId: "r-ck", + taskIdentifier: "task/my-task", + orgId: "o1234", + projectId: "p1234", + environmentId: authenticatedEnvDev.id, + environmentType: "DEVELOPMENT", + queue: "task/my-task", + concurrencyKey: "tenant-1", + timestamp: Date.now(), + eligibleAtMs: Date.now() - 300, + attempt: 0, + }; + + try { + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message, + workerQueue: authenticatedEnvDev.id, + }); + await setTimeout(1000); + const dequeued = await queue.dequeueMessageFromWorkerQueue("c1", authenticatedEnvDev.id); + expect(dequeued?.messageId).toBe(message.runId); + + const entries = await waitForEntries(redis, definition, (es) => + es.some( + (e) => e.fields.op === "gauge" && e.fields.q.includes(":ck:") && e.fields.thr === "0" + ) + ); + const gauges = entries.filter((e) => e.fields.op === "gauge"); + expect(gauges.length).toBeGreaterThan(0); + // The aggregate CK dequeue gauge targets the CK wildcard and never sets thr. + const aggregate = gauges.find((e) => e.fields.q.includes(":ck:") && e.fields.thr === "0"); + assertGauge(aggregate); + expect(Number(aggregate!.fields.ql)).toBeGreaterThanOrEqual(0); + expect(Number(aggregate!.fields.cc)).toBeGreaterThanOrEqual(0); + + // Every CK-path gauge carries the CK-health tail; the enqueue-time reading (and the + // pre-dequeue aggregate reading) sees the backlogged key. + const ckGauges = gauges.filter((e) => e.fields.q.includes(":ck:")); + for (const g of ckGauges) { + expect(g.fields.ckq).toBeDefined(); + expect(g.fields.ckw).toBeDefined(); + expect(Number(g.fields.ckw)).toBeGreaterThanOrEqual(0); + } + expect(ckGauges.some((g) => Number(g.fields.ckq) >= 1)).toBe(true); + + // CK counter entries carry both odometers: the reading has cum + ck/ckcum, and each + // odometer seeds its own baseline entry (cum-only vs ck+ckcum-only). + const enqueues = entries.filter((e) => e.fields.op === "enqueue"); + const reading = enqueues.find((e) => e.fields.cum != null && e.fields.ckcum != null); + expect(reading).toBeDefined(); + expect(reading!.fields.ck).toBe("tenant-1"); + expect(reading!.fields.q).not.toContain(":ck:"); + expect(Number(reading!.fields.cum)).toBe(1); + expect(Number(reading!.fields.ckcum)).toBe(1); + const baseBaseline = enqueues.find((e) => e.fields.cum === "0" && e.fields.ck == null); + expect(baseBaseline).toBeDefined(); + const ckBaseline = enqueues.find((e) => e.fields.ckcum === "0" && e.fields.cum == null); + expect(ckBaseline).toBeDefined(); + expect(ckBaseline!.fields.ck).toBe("tenant-1"); + } finally { + await queue.quit(); + await emitter.close(); + } + }); + + redisTest("gauge sampling gates gauges but not counters", async ({ redisContainer }) => { + const redis = { + keyPrefix: "runqueue:test:", + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }; + const definition: MetricDefinition = { + name: `qm_sample_${Date.now()}`, + shardCount: 2, + consumerGroup: "cg", + maxLen: 1000, + }; + // gaugeSampleRate 0 => sampledSync() always false => Lua gauge never fires; counters still do. + const emitter = new MetricsStreamEmitter({ + redis, + definition, + flag: { enabled: () => true }, + gaugeSampleRate: 0, + }); + const queue = new RunQueue({ + name: "rq", + tracer: trace.getTracer("rq"), + defaultEnvConcurrency: 25, + logger: new Logger("RunQueue", "error"), + keys: new RunQueueFullKeyProducer(), + queueSelectionStrategy: new FairQueueSelectionStrategy({ + redis, + keys: new RunQueueFullKeyProducer(), + }), + redis, + queueMetrics: emitter, + }); + + const message: InputPayload = { + runId: "r-sample", + taskIdentifier: "task/my-task", + orgId: "o1234", + projectId: "p1234", + environmentId: authenticatedEnvDev.id, + environmentType: "DEVELOPMENT", + queue: "task/my-task", + timestamp: Date.now(), + attempt: 0, + }; + + try { + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message, + workerQueue: authenticatedEnvDev.id, + }); + await setTimeout(1000); + await queue.dequeueMessageFromWorkerQueue("c1", authenticatedEnvDev.id); + + // Poll until the counter (enqueue) lands; by then a gauge would have too, if sampled in. + const entries = await waitForEntries(redis, definition, (es) => + es.some((e) => e.fields.op === "enqueue") + ); + expect(entries.some((e) => e.fields.op === "gauge")).toBe(false); + expect(entries.some((e) => e.fields.op === "enqueue")).toBe(true); + } finally { + await queue.quit(); + await emitter.close(); + } + }); +}); + +function assertGauge(gauge: unknown): asserts gauge { + if (!gauge) throw new Error("expected a gauge entry"); +} diff --git a/internal-packages/run-engine/src/run-queue/tests/ckIndex.test.ts b/internal-packages/run-engine/src/run-queue/tests/ckIndex.test.ts index 224540f4efb..4eb47d59bc0 100644 --- a/internal-packages/run-engine/src/run-queue/tests/ckIndex.test.ts +++ b/internal-packages/run-engine/src/run-queue/tests/ckIndex.test.ts @@ -471,4 +471,46 @@ describe("CK Index", () => { await queue.quit(); } }); + + redisTest( + "concurrencyKeyBreakdown lists backlogged keys most-starved first", + async ({ redisContainer }) => { + const queue = createQueue(redisContainer); + try { + const now = Date.now(); + const enqueue = (runId: string, concurrencyKey: string, timestamp: number) => + queue.enqueueMessage({ + env: authenticatedEnvDev, + message: makeMessage({ runId, concurrencyKey, timestamp }), + workerQueue: authenticatedEnvDev.id, + skipDequeueProcessing: true, + }); + + // ck-a has the oldest head (most starved) and 2 queued; ck-b has 1. + await enqueue("r1", "ck-a", now - 10_000); + await enqueue("r2", "ck-a", now - 5_000); + await enqueue("r3", "ck-b", now - 2_000); + + const breakdown = await queue.concurrencyKeyBreakdown(authenticatedEnvDev, "task/my-task"); + expect(breakdown.totalBackloggedKeys).toBe(2); + expect(breakdown.keys).toEqual([ + { concurrencyKey: "ck-a", queued: 2, running: 0, oldestEnqueuedAt: now - 10_000 }, + { concurrencyKey: "ck-b", queued: 1, running: 0, oldestEnqueuedAt: now - 2_000 }, + ]); + + const limited = await queue.concurrencyKeyBreakdown(authenticatedEnvDev, "task/my-task", { + limit: 1, + }); + expect(limited.totalBackloggedKeys).toBe(2); + expect(limited.keys).toHaveLength(1); + expect(limited.keys[0]!.concurrencyKey).toBe("ck-a"); + + // Queues with no CK backlog return an empty breakdown. + const empty = await queue.concurrencyKeyBreakdown(authenticatedEnvDev, "task/other-task"); + expect(empty).toEqual({ totalBackloggedKeys: 0, keys: [] }); + } finally { + await queue.quit(); + } + } + ); }); diff --git a/internal-packages/run-engine/src/run-queue/types.ts b/internal-packages/run-engine/src/run-queue/types.ts index 0905f3971de..8a7d3c93ec5 100644 --- a/internal-packages/run-engine/src/run-queue/types.ts +++ b/internal-packages/run-engine/src/run-queue/types.ts @@ -13,6 +13,9 @@ export const InputPayload = z.object({ queue: z.string(), concurrencyKey: z.string().optional(), timestamp: z.number(), + // Unix ms the run became eligible (delayUntil if set, else triggered-at), pre-priority. + // Dequeue scheduling delay = dequeueTime - eligibleAtMs. Optional for old-payload compat. + eligibleAtMs: z.number().optional(), attempt: z.number(), /** TTL expiration timestamp (unix ms). If set, run will be expired when this time is reached. */ ttlExpiresAt: z.number().optional(), diff --git a/internal-packages/tsql/src/index.test.ts b/internal-packages/tsql/src/index.test.ts index f9aca2f236d..ce358e6ac08 100644 --- a/internal-packages/tsql/src/index.test.ts +++ b/internal-packages/tsql/src/index.test.ts @@ -231,6 +231,26 @@ describe("injectFallbackConditions", () => { expect(modified.where.expression_type).toBe("and"); } }); + + it("should inject into a FROM subquery, where the fallback column's table lives", () => { + const ast = parseTSQLSelect( + "SELECT t, sum(total) AS total FROM (SELECT time AS t, status, count(*) AS total FROM task_runs GROUP BY t, status) GROUP BY t" + ); + const fallbacks: Record<string, WhereClauseCondition> = { + time: { op: "gte", value: "2024-01-01" }, + }; + + const modified = injectFallbackConditions(ast, fallbacks); + expect(modified.expression_type).toBe("select_query"); + if (modified.expression_type === "select_query") { + expect(modified.where).toBeUndefined(); + const inner = modified.select_from?.table; + expect(inner?.expression_type).toBe("select_query"); + if (inner?.expression_type === "select_query") { + expect(isColumnReferencedInExpression(inner.where, "time")).toBe(true); + } + } + }); }); describe("compileTSQL with whereClauseFallback", () => { diff --git a/internal-packages/tsql/src/index.ts b/internal-packages/tsql/src/index.ts index 1d8759c108c..1ebd1a60a5d 100644 --- a/internal-packages/tsql/src/index.ts +++ b/internal-packages/tsql/src/index.ts @@ -429,6 +429,24 @@ export function injectFallbackConditions( // Handle SelectQuery const selectQuery = ast as SelectQuery; + + // When the FROM is a subquery, the fallback columns belong to the inner query's + // table, not this level; descend so e.g. a time fallback lands next to the table ref. + const fromTable = selectQuery.select_from?.table; + if ( + fromTable && + (fromTable.expression_type === "select_query" || + fromTable.expression_type === "select_set_query") + ) { + return { + ...selectQuery, + select_from: { + ...selectQuery.select_from!, + table: injectFallbackConditions(fromTable, fallbacks) as SelectQuery | SelectSetQuery, + }, + }; + } + const existingWhere = selectQuery.where; // Collect fallback expressions for columns not already in WHERE @@ -541,6 +559,12 @@ export interface CompileTSQLOptions { * ``` */ timeRange?: TimeRange; + /** + * Opt-in: emit rows for empty time buckets in a top-level time-bucketed query. + * Counters zero-fill, gauges (columns with `fillMode: "carry"`) carry forward. + * Off by default; output is unchanged when not set. + */ + fillGaps?: boolean; } /** @@ -599,6 +623,7 @@ export function compileTSQL(query: string, options: CompileTSQLOptions): PrintRe fieldMappings: options.fieldMappings, enforcedWhereClause, timeRange: options.timeRange, + fillGaps: options.fillGaps, }); // 6. Print the AST to ClickHouse SQL (enforced conditions applied at printer level) diff --git a/internal-packages/tsql/src/query/functions.ts b/internal-packages/tsql/src/query/functions.ts index 2f2b9278454..a6dadf0f609 100644 --- a/internal-packages/tsql/src/query/functions.ts +++ b/internal-packages/tsql/src/query/functions.ts @@ -645,11 +645,24 @@ export const TSQL_AGGREGATIONS: Record<string, TSQLFunctionMeta> = { maxParams: 1, aggregate: true, }, + quantilesTDigestMerge: { + clickhouseName: "quantilesTDigestMerge", + minArgs: 1, + maxArgs: 1, + minParams: 1, + aggregate: true, + }, sumMerge: { clickhouseName: "sumMerge", minArgs: 1, maxArgs: 1, aggregate: true }, avgMerge: { clickhouseName: "avgMerge", minArgs: 1, maxArgs: 1, aggregate: true }, countMerge: { clickhouseName: "countMerge", minArgs: 1, maxArgs: 1, aggregate: true }, minMerge: { clickhouseName: "minMerge", minArgs: 1, maxArgs: 1, aggregate: true }, maxMerge: { clickhouseName: "maxMerge", minArgs: 1, maxArgs: 1, aggregate: true }, + deltaSumTimestampMerge: { + clickhouseName: "deltaSumTimestampMerge", + minArgs: 1, + maxArgs: 1, + aggregate: true, + }, // Statistical functions simpleLinearRegression: { diff --git a/internal-packages/tsql/src/query/printer.test.ts b/internal-packages/tsql/src/query/printer.test.ts index 30d7b3ef82c..c3a95fde325 100644 --- a/internal-packages/tsql/src/query/printer.test.ts +++ b/internal-packages/tsql/src/query/printer.test.ts @@ -3914,3 +3914,388 @@ describe("timeBucket()", () => { }); }); }); + +// ============================================================ +// fillGaps Tests +// ============================================================ + +describe("timeBucket() fillGaps", () => { + // Schema with a gauge column (fillMode: "carry"), a counter, and a groupable dim. + const metricsSchema: TableSchema = { + name: "metrics", + clickhouseName: "trigger_dev.queue_metrics_v1", + timeConstraint: "bucket_at", + columns: { + bucket_at: { name: "bucket_at", clickhouseName: "created_at", ...column("DateTime64") }, + queue_name: { name: "queue_name", ...column("String") }, + max_running: { name: "max_running", ...column("UInt64"), fillMode: "carry" }, + enqueued: { name: "enqueued", ...column("UInt64"), fillMode: "zero" }, + organization_id: { name: "organization_id", ...column("String") }, + project_id: { name: "project_id", ...column("String") }, + environment_id: { name: "environment_id", ...column("String") }, + }, + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + }; + + // 7-day range -> 6 HOUR buckets (same as the timeBucket() block). + const sevenDayRange = { + from: new Date("2024-01-01T00:00:00Z"), + to: new Date("2024-01-08T00:00:00Z"), + }; + + function ctx(fillGaps: boolean): PrinterContext { + return createPrinterContext({ + schema: createSchemaRegistry([metricsSchema]), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test123" }, + project_id: { op: "eq", value: "proj_test456" }, + environment_id: { op: "eq", value: "env_test789" }, + }, + timeRange: sevenDayRange, + fillGaps, + }); + } + + function run(query: string, fillGaps: boolean) { + const context = ctx(fillGaps); + const result = printToClickHouse(parseTSQLSelect(query), context); + return { ...result, warnings: context.warnings }; + } + + it("emits no WITH FILL when fillGaps is off (unchanged)", () => { + const query = + "SELECT timeBucket(), max(max_running), count() FROM metrics GROUP BY timeBucket ORDER BY timeBucket"; + const { sql } = run(query, false); + expect(sql).not.toContain("WITH FILL"); + expect(sql).not.toContain("INTERPOLATE"); + }); + + it("single-series gauge + counter: WITH FILL plus INTERPOLATE for the gauge only", () => { + const query = + "SELECT timeBucket(), max(max_running) AS max_running, count() AS runs FROM metrics GROUP BY timeBucket ORDER BY timeBucket"; + const { sql, params } = run(query, true); + + // STEP matches the 6 HOUR bucket interval, FROM/TO snapped + parameterized. + expect(sql).toContain("WITH FILL FROM toStartOfInterval({"); + expect(sql).toContain("STEP INTERVAL 6 HOUR"); + expect(sql).toMatch(/TO toStartOfInterval\(\{[^}]+: DateTime64\(6\)\}, INTERVAL 6 HOUR\)/); + + // Gauge carried forward; counter omitted (defaults to 0). + expect(sql).toContain("INTERPOLATE (max_running AS max_running)"); + expect(sql).not.toContain("runs AS runs"); + + // FROM/TO bounds are real parameters carrying the time range. + const dateParams = Object.values(params).filter((v) => v instanceof Date); + expect(dateParams).toContainEqual(sevenDayRange.from); + expect(dateParams).toContainEqual(sevenDayRange.to); + }); + + it("single-series counter only: WITH FILL but no INTERPOLATE", () => { + const query = + "SELECT timeBucket(), count() AS runs FROM metrics GROUP BY timeBucket ORDER BY timeBucket"; + const { sql } = run(query, true); + expect(sql).toContain("WITH FILL FROM toStartOfInterval({"); + expect(sql).toContain("STEP INTERVAL 6 HOUR"); + expect(sql).not.toContain("INTERPOLATE"); + }); + + it("grouped counter only: group dim first, then WITH FILL, no INTERPOLATE", () => { + const query = + "SELECT timeBucket(), queue_name, count() AS runs FROM metrics GROUP BY timeBucket, queue_name ORDER BY timeBucket"; + const { sql } = run(query, true); + expect(sql).toMatch(/ORDER BY queue_name, timebucket ASC WITH FILL/); + expect(sql).toContain("STEP INTERVAL 6 HOUR"); + expect(sql).not.toContain("INTERPOLATE"); + }); + + it("grouped + carry gauge: per-group LOCF via window functions, no INTERPOLATE", () => { + const query = + "SELECT timeBucket(), queue_name, max(max_running) AS max_running FROM metrics GROUP BY timeBucket, queue_name ORDER BY timeBucket"; + const { sql, warnings } = run(query, true); + + // Inner query densifies per group (dims first, then the bucket WITH FILL) + sentinel. + expect(sql).toMatch(/ORDER BY queue_name, timebucket ASC WITH FILL/); + expect(sql).toContain("STEP INTERVAL 6 HOUR"); + expect(sql).toContain("1 AS __tsql_present"); + + // Block id increments at each real row, partitioned by the group dim. + expect(sql).toContain( + "sum(__tsql_present) OVER (PARTITION BY queue_name ORDER BY timebucket ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS __tsql_block" + ); + + // Gauge carried within each (group, block); never INTERPOLATE (which bleeds across groups). + expect(sql).toContain( + "max(if(__tsql_present = 1, max_running, NULL)) OVER (PARTITION BY queue_name, __tsql_block) AS max_running" + ); + expect(sql).not.toContain("INTERPOLATE"); + + // Final result re-ordered by the user's ORDER BY, and not skipped. + expect(sql).toMatch(/\)\s*ORDER BY timebucket ASC$/); + expect(warnings.some((w) => w.code === "fill_skipped_grouped_gauge")).toBe(false); + }); + + it("grouped + carry gauge with a non-plain group dim: fill is skipped", () => { + const query = + "SELECT timeBucket(), upper(queue_name) AS q, max(max_running) AS max_running FROM metrics GROUP BY timeBucket, upper(queue_name) ORDER BY timeBucket"; + const { sql, warnings } = run(query, true); + expect(sql).not.toContain("WITH FILL"); + expect(sql).not.toContain("__tsql_block"); + expect(warnings.some((w) => w.code === "fill_skipped_grouped_gauge")).toBe(true); + }); + + it("user ORDER BY not led by timeBucket: fill is skipped", () => { + const query = + "SELECT timeBucket(), count() AS runs FROM metrics GROUP BY timeBucket ORDER BY runs DESC"; + const { sql } = run(query, true); + expect(sql).not.toContain("WITH FILL"); + expect(sql).not.toContain("INTERPOLATE"); + }); + + it("bucket-led ORDER BY DESC: fill is skipped (ascending fill would be invalid)", () => { + const query = + "SELECT timeBucket(), count() AS runs FROM metrics GROUP BY timeBucket ORDER BY timeBucket DESC"; + const { sql } = run(query, true); + expect(sql).not.toContain("WITH FILL"); + expect(sql).not.toContain("INTERPOLATE"); + // The plain descending order still stands. + expect(sql).toContain("ORDER BY timebucket DESC"); + }); +}); + +describe("cross-queue counter totals via subquery (env-wide throughput shape)", () => { + // deltaSumTimestamp states must merge per queue, then sum outside; this is the + // supported shape for env-wide totals. + const metricsSchema: TableSchema = { + name: "metrics", + clickhouseName: "trigger_dev.queue_metrics_v1", + timeConstraint: "bucket_at", + columns: { + bucket_at: { name: "bucket_at", clickhouseName: "created_at", ...column("DateTime64") }, + queue_name: { name: "queue_name", ...column("String") }, + started_delta: { + name: "started_delta", + mergeGroupKey: "queue_name", + ...column("String"), + groupable: false, + sortable: false, + filterable: false, + }, + organization_id: { name: "organization_id", ...column("String") }, + project_id: { name: "project_id", ...column("String") }, + environment_id: { name: "environment_id", ...column("String") }, + }, + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + }; + + function runSubquery(query: string) { + const context = createPrinterContext({ + schema: createSchemaRegistry([metricsSchema]), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test123" }, + }, + timeRange: { + from: new Date("2024-01-01T00:00:00Z"), + to: new Date("2024-01-08T00:00:00Z"), + }, + }); + const result = printToClickHouse(parseTSQLSelect(query), context); + return { ...result, warnings: context.warnings }; + } + + it("compiles per-queue merge + outer sum, with tenant scoping inside the subquery", () => { + const { sql, params } = runSubquery(` + SELECT t, sum(started) AS started + FROM ( + SELECT timeBucket() AS t, queue_name, deltaSumTimestampMerge(started_delta) AS started + FROM metrics + GROUP BY t, queue_name + ) + GROUP BY t + ORDER BY t + `); + + expect(sql).toContain("deltaSumTimestampMerge(started_delta)"); + expect(sql).toContain("toStartOfInterval(created_at, INTERVAL 6 HOUR)"); + const subqueryStart = sql.indexOf("FROM ("); + const tenantFilter = sql.indexOf("organization_id"); + expect(subqueryStart).toBeGreaterThan(-1); + expect(tenantFilter).toBeGreaterThan(subqueryStart); + expect(Object.values(params)).toContain("org_test123"); + }); +}); + +describe("mergeGroupKey validation", () => { + const metricsSchema: TableSchema = { + name: "metrics", + clickhouseName: "trigger_dev.queue_metrics_v1", + timeConstraint: "bucket_at", + columns: { + bucket_at: { name: "bucket_at", ...column("DateTime64") }, + queue: { name: "queue", clickhouseName: "queue_name", ...column("String") }, + started_delta: { + name: "started_delta", + mergeGroupKey: "queue", + ...column("String"), + groupable: false, + sortable: false, + filterable: false, + }, + organization_id: { name: "organization_id", ...column("String") }, + project_id: { name: "project_id", ...column("String") }, + environment_id: { name: "environment_id", ...column("String") }, + }, + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + }; + + function compile( + query: string, + enforced: Record<string, unknown> = { organization_id: { op: "eq", value: "org_x" } } + ) { + const context = createPrinterContext({ + schema: createSchemaRegistry([metricsSchema]), + enforcedWhereClause: enforced as never, + timeRange: { + from: new Date("2024-01-01T00:00:00Z"), + to: new Date("2024-01-08T00:00:00Z"), + }, + }); + return printToClickHouse(parseTSQLSelect(query), context); + } + + it("rejects an ungrouped, unpinned merge with an actionable message", () => { + expect(() => + compile( + "SELECT timeBucket() AS t, deltaSumTimestampMerge(started_delta) AS started FROM metrics GROUP BY t" + ) + ).toThrowError( + /Merging 'started_delta' across every queue[\s\S]*GROUP BY queue\)[\s\S]*WHERE queue = 'my-queue'[\s\S]*inner GROUP BY t, queue and outer GROUP BY t/ + ); + }); + + it("allows the merge when queue is in the GROUP BY", () => { + const { sql } = compile( + "SELECT timeBucket() AS t, queue, deltaSumTimestampMerge(started_delta) AS started FROM metrics GROUP BY t, queue" + ); + expect(sql).toContain("deltaSumTimestampMerge(started_delta)"); + }); + + it("allows the merge when queue is pinned by an equality filter", () => { + const { sql } = compile( + "SELECT deltaSumTimestampMerge(started_delta) AS started FROM metrics WHERE queue = 'emails'" + ); + expect(sql).toContain("deltaSumTimestampMerge(started_delta)"); + }); + + it("allows the merge when the enforced clause pins queue to one value", () => { + const { sql } = compile( + "SELECT deltaSumTimestampMerge(started_delta) AS started FROM metrics", + { organization_id: { op: "eq", value: "org_x" }, queue: { op: "in", values: ["emails"] } } + ); + expect(sql).toContain("deltaSumTimestampMerge(started_delta)"); + }); + + it("rejects the merge when the enforced clause spans several queues", () => { + expect(() => + compile("SELECT deltaSumTimestampMerge(started_delta) AS started FROM metrics", { + organization_id: { op: "eq", value: "org_x" }, + queue: { op: "in", values: ["emails", "webhooks"] }, + }) + ).toThrowError(/only combine correctly within one queue/); + }); + + it("allows a grouped inner merge summed by the outer query", () => { + const { sql } = compile( + "SELECT t, sum(started) AS started FROM (SELECT timeBucket() AS t, queue, deltaSumTimestampMerge(started_delta) AS started FROM metrics GROUP BY t, queue) GROUP BY t ORDER BY t" + ); + expect(sql).toContain("GROUP BY t, queue_name"); + }); + + it("rejects an ungrouped merge inside a subquery", () => { + expect(() => + compile( + "SELECT t, sum(started) AS started FROM (SELECT timeBucket() AS t, deltaSumTimestampMerge(started_delta) AS started FROM metrics GROUP BY t) GROUP BY t" + ) + ).toThrowError(/only combine correctly within one queue/); + }); +}); + +describe("compound mergeGroupKey validation", () => { + const byKeySchema: TableSchema = { + name: "metrics_by_key", + clickhouseName: "trigger_dev.queue_metrics_ck_v1", + timeConstraint: "bucket_at", + columns: { + bucket_at: { name: "bucket_at", ...column("DateTime64") }, + queue: { name: "queue", clickhouseName: "queue_name", ...column("String") }, + concurrency_key: { name: "concurrency_key", ...column("String") }, + started_delta: { + name: "started_delta", + mergeGroupKey: ["queue", "concurrency_key"], + ...column("String"), + groupable: false, + sortable: false, + filterable: false, + }, + organization_id: { name: "organization_id", ...column("String") }, + project_id: { name: "project_id", ...column("String") }, + environment_id: { name: "environment_id", ...column("String") }, + }, + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + }; + + function compile(query: string) { + const context = createPrinterContext({ + schema: createSchemaRegistry([byKeySchema]), + enforcedWhereClause: { organization_id: { op: "eq", value: "org_x" } } as never, + timeRange: { + from: new Date("2024-01-01T00:00:00Z"), + to: new Date("2024-01-08T00:00:00Z"), + }, + }); + return printToClickHouse(parseTSQLSelect(query), context); + } + + it("requires EVERY listed key grouped or pinned", () => { + expect(() => + compile( + "SELECT deltaSumTimestampMerge(started_delta) AS started FROM metrics_by_key WHERE queue = 'emails'" + ) + ).toThrowError(/only combine correctly within one concurrency_key/); + expect(() => + compile( + "SELECT concurrency_key, deltaSumTimestampMerge(started_delta) AS started FROM metrics_by_key GROUP BY concurrency_key" + ) + ).toThrowError(/only combine correctly within one queue/); + }); + + it("allows pin + group combinations covering both keys", () => { + const grouped = compile( + "SELECT concurrency_key, deltaSumTimestampMerge(started_delta) AS started FROM metrics_by_key WHERE queue = 'emails' GROUP BY concurrency_key" + ); + expect(grouped.sql).toContain("deltaSumTimestampMerge(started_delta)"); + const pinned = compile( + "SELECT deltaSumTimestampMerge(started_delta) AS started FROM metrics_by_key WHERE queue = 'emails' AND concurrency_key = 't1'" + ); + expect(pinned.sql).toContain("deltaSumTimestampMerge(started_delta)"); + const bothGrouped = compile( + "SELECT queue, concurrency_key, deltaSumTimestampMerge(started_delta) AS started FROM metrics_by_key GROUP BY queue, concurrency_key" + ); + expect(bothGrouped.sql).toContain("GROUP BY queue_name, concurrency_key"); + }); +}); diff --git a/internal-packages/tsql/src/query/printer.ts b/internal-packages/tsql/src/query/printer.ts index 00ff8eabdba..ed59b9977fe 100644 --- a/internal-packages/tsql/src/query/printer.ts +++ b/internal-packages/tsql/src/query/printer.ts @@ -385,6 +385,8 @@ export class ClickHousePrinter { nextJoin = nextJoin.next_join; } + this.validateMergeScopedColumns(node); + // Extract SELECT column aliases BEFORE visiting columns // This allows ORDER BY/HAVING to reference aliased columns const savedAliases = this.selectAliases; @@ -459,6 +461,25 @@ export class ClickHousePrinter { this.inProjectionContext = false; } + // Opt-in gap-fill: emit rows for empty time buckets via WITH FILL / INTERPOLATE. + // No-op unless enabled, top-level, and the query is fill-eligible. + let interpolateClause: string | null = null; + let groupedFillWrap: ((inner: string) => string) | null = null; + if (this.context.fillGaps && isTopLevelQuery) { + const fill = this.buildGapFill(node, orderBy, groupBy); + if (fill) { + orderBy = fill.orderBy; + if (fill.kind === "inline") { + interpolateClause = fill.interpolate; + } else { + // Grouped per-group LOCF: add the `present` sentinel to this (now inner) query + // and wrap the rendered SQL in the block-id + carry window layers below. + columns.push(fill.presentColumn); + groupedFillWrap = fill.wrap; + } + } + } + // Process ARRAY JOIN let arrayJoin = ""; if (node.array_join_op) { @@ -487,6 +508,8 @@ export class ClickHousePrinter { having ? `HAVING${space}${having}` : null, windowClause ? `WINDOW${space}${windowClause}` : null, orderBy && orderBy.length > 0 ? `ORDER BY${space}${orderBy.join(comma)}` : null, + // INTERPOLATE must follow the full ORDER BY (including WITH FILL) + interpolateClause, ]; // Process LIMIT @@ -549,6 +572,11 @@ export class ClickHousePrinter { response = this.pretty ? `(${response.trim()})` : `(${response})`; } + // Grouped per-group gap fill wraps this query in the block-id + carry window layers. + if (groupedFillWrap) { + response = groupedFillWrap(response); + } + // Restore saved contexts (for nested queries) this.selectAliases = savedAliases; this.queryHasGroupBy = savedQueryHasGroupBy; @@ -559,6 +587,183 @@ export class ClickHousePrinter { return response; } + /** + * Build the gap-fill transformation (WITH FILL + optional INTERPOLATE) for a + * top-level time-bucketed query. Returns null when the query is not + * fill-eligible (correct-by-construction: emit nothing extra rather than risk + * wrong values). + * + * Eligibility: exactly one timeBucket() column in SELECT, and ORDER BY led by + * that timeBucket column. Carry (gauge) columns are LOCF'd via INTERPOLATE; + * counters zero-fill via WITH FILL's default. Grouped gauge queries are unsafe + * (INTERPOLATE bleeds across groups) and are skipped with a warning. + */ + private buildGapFill( + node: SelectQuery, + orderBy: string[] | null, + groupBy: string[] | null + ): + | { kind: "inline"; orderBy: string[]; interpolate: string | null } + | { kind: "wrap"; orderBy: string[]; presentColumn: string; wrap: (inner: string) => string } + | null { + if (!orderBy || orderBy.length === 0 || !node.select || node.select.length === 0) { + return null; + } + + const timeRange = this.context.timeRange; + if (!timeRange) { + return null; + } + + // Need a time-constraint table to derive the bucket column + interval. + const tableWithConstraint = this.findTimeConstraintTable(); + if (!tableWithConstraint) { + return null; + } + const { tableSchema, clickhouseColumnName } = tableWithConstraint; + const interval = calculateTimeBucketInterval( + timeRange.from, + timeRange.to, + tableSchema.timeBucketThresholds + ); + const bucketSql = `toStartOfInterval(${escapeClickHouseIdentifier(clickhouseColumnName)}, INTERVAL ${interval.value} ${interval.unit})`; + + // Find exactly one timeBucket() column in SELECT and its output alias. + let bucketAlias: string | null = null; + let bucketCount = 0; + for (const col of node.select) { + const inner = (col as Alias).expression_type === "alias" ? (col as Alias).expr : col; + if ( + (inner as Call).expression_type === "call" && + (inner as Call).name.toLowerCase() === "timebucket" + ) { + bucketCount++; + bucketAlias = + (col as Alias).expression_type === "alias" ? (col as Alias).alias : "timebucket"; + } + } + if (bucketCount !== 1 || !bucketAlias) { + return null; + } + + // ORDER BY must be led by the timeBucket column (alias or full expression). + // Don't fight a user ordering like `ORDER BY count DESC`. + const leadTerm = orderBy[0]; + // Strip a trailing ASC/DESC direction without a regex: an unanchored `\s+` before the + // keyword backtracks polynomially across start positions on whitespace runs (CodeQL + // js/polynomial-redos). endsWith + slice is linear. + const trimmedLead = leadTerm.trim(); + const upperLead = trimmedLead.toUpperCase(); + const isDescending = upperLead.endsWith(" DESC"); + const leadExpr = upperLead.endsWith(" ASC") + ? trimmedLead.slice(0, -4).trimEnd() + : isDescending + ? trimmedLead.slice(0, -5).trimEnd() + : trimmedLead; + const matchesBucket = (expr: string): boolean => + expr.toLowerCase() === bucketAlias!.toLowerCase() || expr === bucketSql; + if (!matchesBucket(leadExpr)) { + return null; + } + // WITH FILL is emitted with ascending bounds and a positive STEP, which is + // only valid for an ascending bucket order. A descending order would need + // swapped bounds and a negative step (newer ClickHouse only), so skip the + // gap-fill rewrite and let the plain descending ORDER BY stand. + if (isDescending) { + return null; + } + + // Group dims = GROUP BY expressions that are NOT the timeBucket column. + const groupDims = (groupBy ?? []).filter((g) => !matchesBucket(g.trim())); + + // Classify each SELECT output column. Carry (gauge) columns survive through + // aliases + value-preserving aggregates (see analyzeSelectColumn). A bare column + // that isn't the bucket is a GROUP BY dimension; everything else is a counter or + // derived value that zero-fills. + const carryAliases: string[] = []; + const dimNames: string[] = []; + const orderedOutputs: Array<{ name: string; carry: boolean }> = []; + for (const col of node.select) { + const { outputName, sourceColumn } = this.analyzeSelectColumn(col); + if (!outputName) continue; + const carry = sourceColumn?.fillMode === "carry"; + orderedOutputs.push({ name: outputName, carry }); + if (carry) carryAliases.push(outputName); + const inner = (col as Alias).expression_type === "alias" ? (col as Alias).expr : col; + if (!matchesBucket(outputName) && (inner as Field).expression_type === "field") { + dimNames.push(outputName); + } + } + + // Snap FROM/TO to the bucket grid and parameterize the bounds. + const fromBound = this.context.addValue(timeRange.from); + const toBound = this.context.addValue(timeRange.to); + const withFill = + `WITH FILL FROM toStartOfInterval(${fromBound}, INTERVAL ${interval.value} ${interval.unit})` + + ` TO toStartOfInterval(${toBound}, INTERVAL ${interval.value} ${interval.unit})` + + ` STEP INTERVAL ${interval.value} ${interval.unit}`; + + const esc = escapeClickHouseIdentifier; + + // Single series: WITH FILL on the bucket + INTERPOLATE the carry columns (LOCF); + // counters omitted from INTERPOLATE so they zero-fill. + if (groupDims.length === 0) { + const newOrderBy = [...orderBy]; + newOrderBy[0] = `${leadTerm} ${withFill}`; + const interpolate = + carryAliases.length > 0 + ? `INTERPOLATE (${carryAliases.map((a) => `${esc(a)} AS ${esc(a)}`).join(", ")})` + : null; + return { kind: "inline", orderBy: newOrderBy, interpolate }; + } + + // Grouped, counters only: per-group zero-fill via WITH FILL ordered by the dims. + if (carryAliases.length === 0) { + return { + kind: "inline", + orderBy: [...groupDims, `${leadTerm} ${withFill}`], + interpolate: null, + }; + } + + // Grouped + gauge: per-group LOCF. INTERPOLATE bleeds across groups, so densify per + // group (WITH FILL + a `present` sentinel that is 0 on filled rows), assign a block id + // that increments at each real row, then carry the block's real value via window max. + // Only safe when every GROUP BY dim is a plain column we can PARTITION BY. + if (dimNames.length !== groupDims.length) { + this.context.addWarning( + "fill_skipped_grouped_gauge", + "fillGaps was skipped: per-group gap fill needs every GROUP BY dimension to be a plain column." + ); + return null; + } + + const userOrderBy = [...orderBy]; + const presentCol = "__tsql_present"; + const blockCol = "__tsql_block"; + const partitionDims = dimNames.map(esc).join(", "); + const blockExpr = + `sum(${esc(presentCol)}) OVER (PARTITION BY ${partitionDims} ORDER BY ${esc(bucketAlias)}` + + ` ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS ${esc(blockCol)}`; + const finalColumns = orderedOutputs.map(({ name, carry }) => + carry + ? `max(if(${esc(presentCol)} = 1, ${esc(name)}, NULL)) OVER (PARTITION BY ${partitionDims}, ${esc( + blockCol + )}) AS ${esc(name)}` + : esc(name) + ); + const finalOrderBy = userOrderBy.length > 0 ? ` ORDER BY ${userOrderBy.join(", ")}` : ""; + const wrap = (inner: string): string => + `SELECT ${finalColumns.join(", ")} FROM (SELECT *, ${blockExpr} FROM (${inner.trim()}))${finalOrderBy}`; + + return { + kind: "wrap", + orderBy: [...dimNames.map(esc), `${leadTerm} ${withFill}`], + presentColumn: `1 AS ${esc(presentCol)}`, + wrap, + }; + } + /** * Extract column aliases from a SELECT expression. * Handles explicit aliases (AS name) and implicit names from aggregations/functions. @@ -1014,11 +1219,12 @@ export class ClickHousePrinter { if ((firstArg as Field).expression_type === "field") { const field = firstArg as Field; const columnInfo = this.resolveFieldToColumn(field.chain); - // Only propagate customRenderType, not the full column schema - if (columnInfo.column?.customRenderType) { + // Propagate customRenderType and fillMode (gauge-ness), not the full column schema + if (columnInfo.column?.customRenderType || columnInfo.column?.fillMode) { sourceColumn = { type: inferredType, customRenderType: columnInfo.column.customRenderType, + fillMode: columnInfo.column.fillMode, }; } } @@ -1679,6 +1885,138 @@ export class ClickHousePrinter { // Note: projectId and environmentId are optional - no validation needed } + /** + * Reject queries that merge a scope-keyed aggregate state column (`mergeGroupKey`) + * across values of its key: such merges silently return wrong numbers. Valid shapes + * group by the key column or pin it to a single value (in the query's WHERE or via + * the enforced clause). Runs per SELECT scope; subqueries validate themselves. + */ + private validateMergeScopedColumns(node: SelectQuery): void { + for (const tableSchema of this.tableContexts.values()) { + for (const column of Object.values(tableSchema.columns)) { + if (!column.mergeGroupKey) continue; + const keys = Array.isArray(column.mergeGroupKey) + ? column.mergeGroupKey + : [column.mergeGroupKey]; + if (!this.scopeReferencesColumn(node, column.name)) continue; + for (const key of keys) { + if (this.groupByIncludesColumn(node, key)) continue; + if (this.wherePinsColumn(node.where, key)) continue; + if (this.enforcedPinsColumn(tableSchema, key)) continue; + throw new QueryError( + `Merging '${column.name}' across every ${key} returns wrong totals: its aggregate ` + + `states are kept per ${key} and only combine correctly within one ${key}. Either ` + + `add '${key}' to the GROUP BY and sum the per-${key} results in an outer query, ` + + `for example: SELECT sum(v) AS total FROM (SELECT ${key}, ` + + `deltaSumTimestampMerge(${column.name}) AS v FROM ${tableSchema.name} ` + + `GROUP BY ${key}). Or filter to a single ${key}, for example: ` + + `WHERE ${key} = 'my-${key}'. For a time series, bucket both layers: ` + + `inner GROUP BY t, ${key} and outer GROUP BY t.` + ); + } + } + } + } + + private scopeReferencesColumn(node: SelectQuery, name: string): boolean { + const parts: unknown[] = [ + node.select, + node.prewhere, + node.where, + node.group_by, + node.having, + node.order_by, + ]; + return parts.some((part) => this.expressionReferencesColumn(part, name)); + } + + private expressionReferencesColumn( + expr: unknown, + name: string, + seen = new WeakSet<object>() + ): boolean { + if (expr === null || typeof expr !== "object") return false; + if (seen.has(expr)) return false; + seen.add(expr); + if (Array.isArray(expr)) { + return expr.some((item) => this.expressionReferencesColumn(item, name, seen)); + } + const candidate = expr as { expression_type?: string; chain?: unknown[] }; + if ( + candidate.expression_type === "select_query" || + candidate.expression_type === "select_set_query" + ) { + return false; + } + if ( + candidate.expression_type === "field" && + Array.isArray(candidate.chain) && + candidate.chain[candidate.chain.length - 1] === name + ) { + return true; + } + return Object.entries(expr).some( + ([property, value]) => + property !== "type" && + property !== "parent" && + this.expressionReferencesColumn(value, name, seen) + ); + } + + private groupByIncludesColumn(node: SelectQuery, name: string): boolean { + return (node.group_by ?? []).some((expr) => { + const field = expr as Field; + return ( + field.expression_type === "field" && + Array.isArray(field.chain) && + field.chain[field.chain.length - 1] === name + ); + }); + } + + // Pins only count on the top-level AND chain: a pin inside an OR guarantees nothing. + private wherePinsColumn(where: Expression | undefined, name: string): boolean { + if (!where) return false; + if (where.expression_type === "and") { + return (where as And).exprs.some((expr) => this.wherePinsColumn(expr, name)); + } + if (where.expression_type !== "compare_operation") return false; + const cmp = where as CompareOperation; + const isKeyField = (side: Expression) => { + const field = side as Field; + return ( + field.expression_type === "field" && + Array.isArray(field.chain) && + field.chain[field.chain.length - 1] === name + ); + }; + const fieldSide = [cmp.left, cmp.right].find(isKeyField); + if (!fieldSide) return false; + if (cmp.op === CompareOperationOp.Eq) return true; + if (cmp.op === CompareOperationOp.In || cmp.op === CompareOperationOp.GlobalIn) { + const other = fieldSide === cmp.left ? cmp.right : cmp.left; + if ((other as Constant).expression_type === "constant") return true; + const tuple = other as Tuple; + return tuple.expression_type === "tuple" && tuple.exprs.length === 1; + } + return false; + } + + private enforcedPinsColumn(tableSchema: TableSchema, key: string): boolean { + const names = [key]; + const clickhouseName = tableSchema.columns[key]?.clickhouseName; + if (clickhouseName) names.push(clickhouseName); + for (const name of names) { + const condition = this.context.enforcedWhereClause[name] as + | { op?: string; values?: unknown[] } + | undefined; + if (!condition) continue; + if (condition.op === "eq") return true; + if (condition.op === "in" && condition.values?.length === 1) return true; + } + return false; + } + /** * Format a Date as a ClickHouse-compatible DateTime64 string. * ClickHouse expects format: 'YYYY-MM-DD HH:MM:SS.mmm' (in UTC) diff --git a/internal-packages/tsql/src/query/printer_context.ts b/internal-packages/tsql/src/query/printer_context.ts index d0fb41b5327..a964e2e04af 100644 --- a/internal-packages/tsql/src/query/printer_context.ts +++ b/internal-packages/tsql/src/query/printer_context.ts @@ -125,6 +125,9 @@ export class PrinterContext { */ readonly timeRange?: TimeRange; + /** When true, time-bucketed queries emit rows for empty buckets (opt-in). */ + readonly fillGaps?: boolean; + constructor( /** Schema registry containing allowed tables and columns */ public readonly schema: SchemaRegistry, @@ -138,13 +141,16 @@ export class PrinterContext { */ enforcedWhereClause: Record<string, WhereClauseCondition> = {}, /** Time range for timeBucket() interval calculation */ - timeRange?: TimeRange + timeRange?: TimeRange, + /** Opt-in gap-fill for time-bucketed queries */ + fillGaps?: boolean ) { // Initialize with default settings this.settings = { ...DEFAULT_QUERY_SETTINGS, ...settings }; this.fieldMappings = fieldMappings; this.enforcedWhereClause = enforcedWhereClause; this.timeRange = timeRange; + this.fillGaps = fillGaps; } /** @@ -225,7 +231,8 @@ export class PrinterContext { this.settings, this.fieldMappings, this.enforcedWhereClause, - this.timeRange + this.timeRange, + this.fillGaps ); // Share the same values map so parameters are unified child.values = this.values; @@ -277,6 +284,8 @@ export interface PrinterContextOptions { * When provided, `timeBucket()` uses this to determine the appropriate bucket size. */ timeRange?: TimeRange; + /** When true, time-bucketed queries emit rows for empty buckets (opt-in). */ + fillGaps?: boolean; } /** @@ -288,6 +297,7 @@ export function createPrinterContext(options: PrinterContextOptions): PrinterCon options.settings, options.fieldMappings, options.enforcedWhereClause, - options.timeRange + options.timeRange, + options.fillGaps ); } diff --git a/internal-packages/tsql/src/query/schema.ts b/internal-packages/tsql/src/query/schema.ts index 9a1e2d2ddfe..0d50c1fbe3c 100644 --- a/internal-packages/tsql/src/query/schema.ts +++ b/internal-packages/tsql/src/query/schema.ts @@ -122,6 +122,18 @@ export interface ColumnSchema { * ``` */ customRenderType?: string; + /** + * Gap-fill behavior when the opt-in `fillGaps` feature emits rows for empty + * time buckets: `"carry"` = gauge (LOCF via INTERPOLATE), `"zero"` (default) + * = counter (missing buckets get 0). + */ + fillMode?: "zero" | "carry"; + /** + * Aggregate-state column whose states only merge correctly within one value of the + * named column(s) (e.g. per-queue counter states). Queries referencing it must GROUP BY + * every listed column or pin each to a single value; other shapes fail to compile. + */ + mergeGroupKey?: string | string[]; /** * Example value for documentation purposes. * @@ -409,6 +421,26 @@ export interface TableSchema { * is needed to get correct results. Not needed for plain MergeTree tables. */ useFinal?: boolean; + /** + * Coarser physical rollups with an identical logical schema, substituted by callers + * (not the printer) when the timeBucket() interval is at least minIntervalSeconds. + */ + rollups?: Array<{ minIntervalSeconds: number; clickhouseName: string }>; + /** + * Opt into the ClickHouse query cache; callers align time bounds to alignSeconds + * so repeated auto-refresh queries share cache entries. + */ + queryCache?: { ttlSeconds: number; alignSeconds: number }; + /** + * Excluded from user-facing listings (query editor, schema docs, schema API) by + * callers; the engine still compiles queries against it. + */ + hidden?: boolean; + /** + * Names the connection pool callers should read this table through, for a read family heavy + * enough to want its own capacity. Callers resolve the name; unset means the default pool. + */ + queryClient?: string; } /** diff --git a/packages/cli-v3/src/apiClient.ts b/packages/cli-v3/src/apiClient.ts index 2b76ae021e1..c972802f4d2 100644 --- a/packages/cli-v3/src/apiClient.ts +++ b/packages/cli-v3/src/apiClient.ts @@ -368,6 +368,49 @@ export class CliApiClient { ); } + /** + * Fetch a server-rendered report (text + sparkline). Thin pass-through. `format`: + * "markdown" (agents/chat) or "ansi" (terminal). Uses this client's env API key. + */ + async getReport( + key: string, + options?: { period?: string; format?: "markdown" | "ansi" } + ): Promise<string> { + if (!this.accessToken) { + throw new Error("getReport: No access token"); + } + + const searchParams = new URLSearchParams({ format: options?.format ?? "markdown" }); + if (options?.period) { + searchParams.set("period", options.period); + } + + const response = await fetch( + `${this.apiURL}/api/v1/reports/${encodeURIComponent(key)}?${searchParams.toString()}`, + { + method: "GET", + headers: this.getHeaders(), + } + ); + + if (!response.ok) { + let bodySnippet = ""; + try { + const text = (await response.text()).trim(); + bodySnippet = text.length > 500 ? `${text.slice(0, 500)}…` : text; + } catch { + // best-effort; ignore + } + throw new Error( + `Failed to fetch report "${key}": ${response.status} ${response.statusText}${ + bodySnippet ? ` — ${bodySnippet}` : "" + }` + ); + } + + return response.text(); + } + async importEnvVars( projectRef: string, slug: string, diff --git a/packages/cli-v3/src/cli/index.ts b/packages/cli-v3/src/cli/index.ts index e9012296553..fc3958fb1d6 100644 --- a/packages/cli-v3/src/cli/index.ts +++ b/packages/cli-v3/src/cli/index.ts @@ -14,6 +14,7 @@ import { configureUpdateCommand } from "../commands/update.js"; import { configureWhoamiCommand } from "../commands/whoami.js"; import { configureMintTokenCommand } from "../commands/mint-token.js"; import { configureMcpCommand } from "../commands/mcp.js"; +import { configureReportCommand } from "../commands/report.js"; import { COMMAND_NAME } from "../consts.js"; import { VERSION } from "../version.js"; import { installExitHandler } from "./common.js"; @@ -42,6 +43,7 @@ configureUpdateCommand(program); configurePreviewCommand(program); configureAnalyzeCommand(program); configureMcpCommand(program); +configureReportCommand(program); configureInstallMcpCommand(program); configureSkillsCommand(program); diff --git a/packages/cli-v3/src/commands/mcp.ts b/packages/cli-v3/src/commands/mcp.ts index 7f94e40a3af..cc5d951eb24 100644 --- a/packages/cli-v3/src/commands/mcp.ts +++ b/packages/cli-v3/src/commands/mcp.ts @@ -10,6 +10,7 @@ import { serverMetadata } from "../mcp/config.js"; import { McpContext } from "../mcp/context.js"; import { toMcpContextOptions } from "../mcp/contextOptions.js"; import { FileLogger } from "../mcp/logger.js"; +import { registerPrompts } from "../mcp/prompts.js"; import { registerTools } from "../mcp/tools.js"; import { printStandloneInitialBanner } from "../utilities/initialBanner.js"; import { logger } from "../utilities/logger.js"; @@ -21,6 +22,7 @@ const McpCommandOptions = CommonCommandOptions.extend({ logFile: z.string().optional(), devOnly: z.boolean().default(false), readonly: z.boolean().default(false), + install: z.boolean().default(false), }); export type McpCommandOptions = z.infer<typeof McpCommandOptions>; @@ -40,6 +42,10 @@ export function configureMcpCommand(program: Command) { "Run in read-only mode. Write tools (deploy, trigger_task, cancel_run) are hidden from the AI." ) .option("--log-file <log file>", "The file to log to") + .option( + "--install", + "Run the interactive install wizard instead of starting the server. Bare `mcp` always starts the server (so clients that spawn it over a PTY don't get stuck in the wizard)." + ) ).action(async (options) => { wrapCommandAction("mcp", McpCommandOptions, options, async (opts) => { await mcpCommand(opts); @@ -48,7 +54,11 @@ export function configureMcpCommand(program: Command) { } export async function mcpCommand(options: McpCommandOptions) { - if (process.stdout.isTTY) { + // The install wizard runs ONLY when explicitly requested (`trigger mcp --install`). + // Bare `trigger mcp` always starts the server — MCP hosts (e.g. Claude Code) spawn it + // over a PTY, so `process.stdout.isTTY` is true even though no human is there; gating + // the wizard on isTTY made the server never start and the client time out. + if (options.install) { await printStandloneInitialBanner(true, options.profile); intro("Welcome to the Trigger.dev MCP server install wizard 🧙"); @@ -98,6 +108,7 @@ export async function mcpCommand(options: McpCommandOptions) { const context = new McpContext(server, toMcpContextOptions(options, fileLogger)); registerTools(context); + registerPrompts(context); await server.connect(transport); } diff --git a/packages/cli-v3/src/commands/report.ts b/packages/cli-v3/src/commands/report.ts new file mode 100644 index 00000000000..9fb46ac8801 --- /dev/null +++ b/packages/cli-v3/src/commands/report.ts @@ -0,0 +1,111 @@ +import { tryCatch } from "@trigger.dev/core/utils"; +import type { Command } from "commander"; +import { z } from "zod"; +import { + CommonCommandOptions, + commonOptions, + handleTelemetry, + wrapCommandAction, +} from "../cli/common.js"; +import { loadConfig } from "../config.js"; +import { ReportPeriodSchema } from "../mcp/schemas.js"; +import { getProjectClient } from "../utilities/session.js"; +import { login } from "./login.js"; + +const ReportOptions = CommonCommandOptions.extend({ + config: z.string().optional(), + projectRef: z.string().optional(), + env: z.enum(["dev", "staging", "prod", "preview", "production"]).default("prod"), + branch: z.string().optional(), + period: ReportPeriodSchema.optional(), +}); + +type ReportOptions = z.infer<typeof ReportOptions>; + +export function configureReportCommand(program: Command) { + return commonOptions( + program + .command("report") + .description( + "Print an interpreted report for an environment. Currently: 'health' — is work flowing, is it your code, is the telemetry fresh." + ) + .argument("[key]", "The report to render", "health") + .option("-c, --config <config file>", "The name of the config file") + .option( + "-p, --project-ref <project ref>", + "The project ref. Required if there is no config file" + ) + .option("-e, --env <env>", "The environment (dev, staging, prod, preview)", "prod") + .option("-b, --branch <branch>", "The preview branch when using --env preview") + .option("--period <period>", "Time window, e.g. 1h (default), 24h, 7d") + ).action(async (key, options) => { + await handleTelemetry(async () => { + await reportCommand(key, options); + }); + }); +} + +async function reportCommand(key: string, options: unknown) { + return await wrapCommandAction("report", ReportOptions, options, async (opts: ReportOptions) => { + await _reportCommand(key, opts); + }); +} + +async function _reportCommand(key: string, options: ReportOptions) { + // Clean output: no banner, just the report (so it can be piped). + const authorization = await login({ + embedded: true, + defaultApiUrl: options.apiUrl, + profile: options.profile, + silent: true, + }); + + if (!authorization.ok) { + throw new Error(`You must login first. Use the \`login\` CLI command.`); + } + + const resolvedConfig = await loadConfig({ + overrides: { project: options.projectRef }, + configFile: options.config, + }); + + const env = options.env === "production" ? "prod" : options.env; + if (env === "preview" && !options.branch) { + throw new Error("Missing branch for the preview environment."); + } + + const projectClient = await getProjectClient({ + accessToken: authorization.auth.accessToken, + apiUrl: authorization.auth.apiUrl, + projectRef: resolvedConfig.project, + env, + branch: options.branch, + profile: options.profile, + }); + + if (!projectClient) { + throw new Error("Failed to get project client"); + } + + // Colour in a real terminal; plain markdown when piped. NO_COLOR / FORCE_COLOR follow the + // de-facto supports-color spec: NO_COLOR (any value) OR FORCE_COLOR=0 disables outright; + // FORCE_COLOR set to anything else force-enables. Both win over isTTY (true even for + // PTY-spawned agents — the mcp trap). + const forceColor = process.env.FORCE_COLOR; + const disabled = "NO_COLOR" in process.env || forceColor === "0"; + const useColor = disabled + ? false + : forceColor !== undefined + ? true + : Boolean(process.stdout.isTTY); + const format = useColor ? "ansi" : "markdown"; + const [error, report] = await tryCatch( + projectClient.client.getReport(key, { period: options.period, format }) + ); + + if (error) { + throw error; + } + + process.stdout.write(report.endsWith("\n") ? report : `${report}\n`); +} diff --git a/packages/cli-v3/src/mcp/config.ts b/packages/cli-v3/src/mcp/config.ts index 31b5c6ec4dd..227b0c5506e 100644 --- a/packages/cli-v3/src/mcp/config.ts +++ b/packages/cli-v3/src/mcp/config.ts @@ -136,6 +136,12 @@ export const toolsMetadata = { description: "Execute a single widget query from a built-in dashboard. Use list_dashboards first to see available dashboards, widget IDs, and their queries. Supports time period and scope options.", }, + get_report: { + name: "get_report", + title: "Get Report", + description: + "Render an interpreted report (an answered question, not a raw panel) as text + sparklines. The server computes the verdict deterministically. Currently: 'health' — whether work is flowing, whether the runs that start are healthy, and whether the telemetry is fresh, with a headline verdict and a suggested next action.", + }, whoami: { name: "whoami", title: "Who Am I", diff --git a/packages/cli-v3/src/mcp/prompts.ts b/packages/cli-v3/src/mcp/prompts.ts new file mode 100644 index 00000000000..6beb6398eae --- /dev/null +++ b/packages/cli-v3/src/mcp/prompts.ts @@ -0,0 +1,64 @@ +import { completable } from "@modelcontextprotocol/sdk/server/completable.js"; +import { z } from "zod"; +import type { McpContext } from "./context.js"; +import { GetReportInput } from "./schemas.js"; + +// Derived from the tool's schema so completion can't drift from what get_report accepts. +// `environment` has a `.default(...)`, so read its enum through `removeDefault()`. +const REPORT_KEYS = GetReportInput.shape.key.options; +const ENVIRONMENTS = GetReportInput.shape.environment.removeDefault().options; + +/** + * MCP prompts surface as slash commands in hosts that support them (Claude Code renders + * this as /mcp__trigger__report), so `/report health` is a real command — no per-project + * files needed. + */ +export function registerPrompts(context: McpContext) { + context.server.registerPrompt( + "report", + { + title: "Report", + description: + "Render an interpreted report for an environment. Currently: 'health' — is work flowing, is it your code, is the data fresh.", + argsSchema: { + key: completable(z.string().optional(), (value) => + REPORT_KEYS.filter((k) => k.startsWith(value ?? "")) + ), + environment: completable(z.string().optional(), (value) => + ENVIRONMENTS.filter((e) => e.startsWith(value ?? "")) + ), + // Plain string on purpose: MCP prompt args must be simple string schemas (the SDK + // introspects them as such), and this only forwards into get_report, which validates + // the period with the shared ReportPeriodSchema. Don't swap in a refined schema here. + period: z.string().optional(), + }, + }, + async ({ key, environment, period }) => { + const reportKey = key || "health"; + const env = environment || (context.options.devOnly ? "dev" : "prod"); + + const args = [`key: "${reportKey}"`, `environment: "${env}"`]; + if (period) { + args.push(`period: "${period}"`); + } + + return { + messages: [ + { + role: "user" as const, + content: { + type: "text" as const, + text: `Fetch the ${reportKey} report by calling the get_report tool with { ${args.join( + ", " + )} } (add projectRef if this workspace has more than one Trigger.dev project). + +Show the returned report to the user EXACTLY as-is, inside a fenced code block: it is monospace-aligned with unicode sparklines, so do not paraphrase, reformat, translate, or trim whitespace. + +After the block, add at most two sentences of your own — and only if the report's recommended action intersects with something you know about this project (for example, where the relevant configuration lives). Otherwise add nothing.`, + }, + }, + ], + }; + } + ); +} diff --git a/packages/cli-v3/src/mcp/schemas.ts b/packages/cli-v3/src/mcp/schemas.ts index 01aa9577cb3..64a03438028 100644 --- a/packages/cli-v3/src/mcp/schemas.ts +++ b/packages/cli-v3/src/mcp/schemas.ts @@ -271,6 +271,50 @@ export const ListDashboardsInput = CommonProjectsInput.pick({ export type ListDashboardsInput = z.output<typeof ListDashboardsInput>; +/** + * Shared period validation for the report surfaces (MCP tool + `trigger report` CLI), so they + * reject garbage/absurd ranges consistently client-side instead of only at the HTTP API. The + * webapp route (`api.v1.reports.$key.ts`) mirrors this regex + bound as the authoritative + * security boundary — it can't import from the CLI, so the two are kept intentionally in sync. + */ +const PERIOD_UNIT_MS: Record<string, number> = { s: 1e3, m: 6e4, h: 36e5, d: 864e5, w: 6048e5 }; +const MAX_PERIOD_MS = 90 * 864e5; // 90d +export const ReportPeriodSchema = z + .string() + .regex(/^[1-9]\d*[smhdw]$/, "period must be a shorthand like '1h', '30m', or '7d'") + .refine( + // The regex guarantees the last char is a known unit; `?? 0` just satisfies the type checker. + (p) => Number(p.slice(0, -1)) * (PERIOD_UNIT_MS[p.slice(-1)] ?? 0) <= MAX_PERIOD_MS, + "period is too large (max 90d)" + ); + +// `environment` inherits CommonProjectsInput's `.default("dev")` — intentional: the MCP server +// is dev-centric (often `--dev-only`), so an unspecified env reports on dev. The `trigger report` +// CLI defaults to prod instead (a manual prod check). Agents should pass `environment` explicitly. +export const GetReportInput = CommonProjectsInput.pick({ + projectRef: true, + configPath: true, + environment: true, + branch: true, +}).extend({ + key: z + .enum(["health"]) + .describe( + "The report to render. 'health' answers 'is work flowing, and is a problem my code or the platform?' with an interpreted verdict (flow / execution / liveness)." + ), + period: ReportPeriodSchema.optional().describe( + "Time period shorthand for the live window, e.g. '1h' (default), '7d'." + ), + color: z + .boolean() + .optional() + .describe( + "Return the report as ANSI-coloured text instead of markdown. Only renders in hosts that display ANSI in tool output." + ), +}); + +export type GetReportInput = z.output<typeof GetReportInput>; + export const RunDashboardQueryInput = CommonProjectsInput.extend({ dashboardKey: z .string() diff --git a/packages/cli-v3/src/mcp/tools.ts b/packages/cli-v3/src/mcp/tools.ts index 1fe7be6256a..8fa9eabf6f8 100644 --- a/packages/cli-v3/src/mcp/tools.ts +++ b/packages/cli-v3/src/mcp/tools.ts @@ -12,6 +12,7 @@ import { startDevServerTool, stopDevServerTool, devServerStatusTool } from "./to import { listPreviewBranchesTool } from "./tools/previewBranches.js"; import { listProfilesTool, switchProfileTool, whoamiTool } from "./tools/profiles.js"; import { getQuerySchemaTool, queryTool } from "./tools/query.js"; +import { getReportTool } from "./tools/report.js"; import { cancelRunTool, getRunDetailsTool, @@ -89,6 +90,7 @@ export function registerTools(context: McpContext) { startAgentChatTool, sendAgentMessageTool, closeAgentChatTool, + getReportTool, ]; for (const tool of tools) { @@ -112,7 +114,9 @@ export function registerTools(context: McpContext) { }, async (input, extra) => { try { - return tool.handler(input, { ...extra, ctx: context }); + // await so a rejected async handler (e.g. a network failure in get_report) is caught + // here and wrapped, not surfaced as an unhandled rejection. + return await tool.handler(input, { ...extra, ctx: context }); } catch (error) { return respondWithError(error); } diff --git a/packages/cli-v3/src/mcp/tools/report.ts b/packages/cli-v3/src/mcp/tools/report.ts new file mode 100644 index 00000000000..d24847f64da --- /dev/null +++ b/packages/cli-v3/src/mcp/tools/report.ts @@ -0,0 +1,45 @@ +import { toolsMetadata } from "../config.js"; +import { GetReportInput } from "../schemas.js"; +import { respondWithError, toolHandler } from "../utils.js"; + +/** + * `get_report` fetches a server-rendered report (verdict + text + sparklines). Plain + * markdown by default, or ANSI when `color` is set (for hosts that display escapes in + * tool output). Read-only; the handler enforces devOnly. + */ +export const getReportTool = { + name: toolsMetadata.get_report.name, + title: toolsMetadata.get_report.title, + description: toolsMetadata.get_report.description, + inputSchema: GetReportInput.shape, + handler: toolHandler(GetReportInput.shape, async (input, { ctx }) => { + ctx.logger?.log("calling get_report", { input }); + + if (ctx.options.devOnly && input.environment !== "dev") { + return respondWithError( + `This MCP server is only available for the dev environment. You tried to access the ${input.environment} environment. Remove the --dev-only flag to access other environments.` + ); + } + + const projectRef = await ctx.getProjectRef({ + projectRef: input.projectRef, + cwd: input.configPath, + }); + + const apiClient = await ctx.getApiClient({ + projectRef, + environment: input.environment, + scopes: ["read:query"], + branch: input.branch, + }); + + const text = await apiClient.getReport(input.key, { + period: input.period, + format: input.color ? "ansi" : "markdown", + }); + + return { + content: [{ type: "text" as const, text }], + }; + }), +}; diff --git a/packages/core/src/v3/apiClient/index.ts b/packages/core/src/v3/apiClient/index.ts index 5329f15fad0..23f13f37573 100644 --- a/packages/core/src/v3/apiClient/index.ts +++ b/packages/core/src/v3/apiClient/index.ts @@ -1997,6 +1997,39 @@ export class ApiClient { ); } + /** + * Fetch a server-rendered report (text + sparkline). Thin pass-through — the string + * is ready to display. `format`: "markdown" (default, agents/chat) or "ansi" (terminal). + */ + async getReport( + key: string, + options?: { period?: string; format?: "markdown" | "ansi" } + ): Promise<string> { + const searchParams = new URLSearchParams({ format: options?.format ?? "markdown" }); + if (options?.period) { + searchParams.set("period", options.period); + } + + const response = await fetch( + `${this.baseUrl}/api/v1/reports/${encodeURIComponent(key)}?${searchParams.toString()}`, + { + method: "GET", + headers: this.#getHeaders(false), + } + ); + + if (!response.ok) { + const bodySnippet = await readBodySnippet(response); + throw new Error( + `Failed to fetch report "${key}": ${response.status} ${response.statusText}${ + bodySnippet ? ` — ${bodySnippet}` : "" + }` + ); + } + + return response.text(); + } + #getHeaders(spanParentAsLink: boolean, additionalHeaders?: Record<string, string | undefined>) { const headers: Record<string, string> = { "Content-Type": "application/json", @@ -2474,6 +2507,22 @@ function sleep(ms: number): Promise<void> { return new Promise((resolve) => setTimeout(resolve, ms)); } +/** + * Best-effort read of a (likely error) response body for inclusion in a thrown Error. + * Never throws, and truncates so we don't dump a huge HTML page into an error message. + */ +async function readBodySnippet(response: Response, maxLength = 500): Promise<string> { + try { + const text = (await response.text()).trim(); + if (!text) { + return ""; + } + return text.length > maxLength ? `${text.slice(0, maxLength)}…` : text; + } catch { + return ""; + } +} + /** * Safely cancels a ReadableStream, handling the case where it might be locked. * diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 486a49560f3..766ab1a8eb4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -293,6 +293,9 @@ importers: '@internal/llm-model-catalog': specifier: workspace:* version: link:../../internal-packages/llm-model-catalog + '@internal/metrics-pipeline': + specifier: workspace:* + version: link:../../internal-packages/metrics-pipeline '@internal/redis': specifier: workspace:* version: link:../../internal-packages/redis @@ -1108,6 +1111,25 @@ importers: specifier: 4.1.7 version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(@vitest/coverage-v8@4.1.7)(vite@6.4.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.22.4)(yaml@2.9.0)) + internal-packages/metrics-pipeline: + dependencies: + '@internal/redis': + specifier: workspace:* + version: link:../redis + '@internal/tracing': + specifier: workspace:* + version: link:../tracing + '@trigger.dev/core': + specifier: workspace:* + version: link:../../packages/core + devDependencies: + '@internal/testcontainers': + specifier: workspace:* + version: link:../testcontainers + rimraf: + specifier: 6.0.1 + version: 6.0.1 + internal-packages/otlp-importer: dependencies: long: @@ -1188,6 +1210,9 @@ importers: '@internal/cache': specifier: workspace:* version: link:../cache + '@internal/metrics-pipeline': + specifier: workspace:* + version: link:../metrics-pipeline '@internal/redis': specifier: workspace:* version: link:../redis From a81ad4949cb6d0f8a02ea3c5bc11b616edefb582 Mon Sep 17 00:00:00 2001 From: Chris Arderne <chris@trigger.dev> Date: Wed, 29 Jul 2026 17:24:00 +0100 Subject: [PATCH 32/67] feat(database,rbac): add multiple environment API key foundations (#4388) Adds the storage model and authorization contracts needed for multiple environment API keys. Credentials are represented by hashed values, revocation and expiration state, and persisted effective scopes. The built-in authorization fallback exposes full-access policy preparation, while optional authorization extensions can supply additional presets and task-aware scope generation. This change does not create, display, or authenticate additional keys. --- .changeset/public-token-expiration-seconds.md | 5 + apps/webapp/app/utils/apiKeys.test.ts | 42 ++++ apps/webapp/app/utils/apiKeys.ts | 51 +++++ .../migration.sql | 30 +++ .../database/prisma/schema.prisma | 27 +++ internal-packages/rbac/src/ability.test.ts | 38 ++++ internal-packages/rbac/src/ability.ts | 2 +- .../rbac/src/apiKeyPolicyDefaults.test.ts | 58 ++++++ internal-packages/rbac/src/fallback.ts | 34 +++- internal-packages/rbac/src/index.ts | 60 +++++- packages/core/src/v3/jwt.ts | 34 ++++ packages/plugins/src/index.ts | 14 +- packages/plugins/src/rbac.ts | 184 ++++++++++++++++-- 13 files changed, 560 insertions(+), 19 deletions(-) create mode 100644 .changeset/public-token-expiration-seconds.md create mode 100644 apps/webapp/app/utils/apiKeys.test.ts create mode 100644 apps/webapp/app/utils/apiKeys.ts create mode 100644 internal-packages/database/prisma/migrations/20260723112558_add_environment_api_keys/migration.sql create mode 100644 internal-packages/rbac/src/apiKeyPolicyDefaults.test.ts diff --git a/.changeset/public-token-expiration-seconds.md b/.changeset/public-token-expiration-seconds.md new file mode 100644 index 00000000000..5eba6ba689b --- /dev/null +++ b/.changeset/public-token-expiration-seconds.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/sdk": patch +--- + +Correct the `expirationTime` docs on `auth.createPublicToken` and the trigger-token helpers: a number is a Unix timestamp in seconds, not milliseconds. diff --git a/apps/webapp/app/utils/apiKeys.test.ts b/apps/webapp/app/utils/apiKeys.test.ts new file mode 100644 index 00000000000..26e7d697da6 --- /dev/null +++ b/apps/webapp/app/utils/apiKeys.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, test } from "vitest"; +import { + apiKeyPrefix, + generateAdditionalApiKey, + generateRootApiKey, + hashApiKey, + obfuscateApiKey, +} from "./apiKeys"; + +describe("API key utilities", () => { + test.each([ + ["DEVELOPMENT", "tr_dev_"], + ["STAGING", "tr_stg_"], + ["PRODUCTION", "tr_prod_"], + ["PREVIEW", "tr_preview_"], + ] as const)("generates %s keys", (environmentType, prefix) => { + const root = generateRootApiKey(environmentType); + const additional = generateAdditionalApiKey(environmentType); + + expect(root.apiKey).toMatch(new RegExp(`^${prefix}[A-Za-z0-9]{24}$`)); + expect(root.keyHash).toBe(hashApiKey(root.apiKey)); + expect(root.lastFour).toBe(root.apiKey.slice(-4)); + expect(additional.apiKey).toMatch(new RegExp(`^${prefix}sk_[A-Za-z0-9]{24}$`)); + expect(additional.keyHash).toBe(hashApiKey(additional.apiKey)); + expect(additional.lastFour).toBe(additional.apiKey.slice(-4)); + expect(apiKeyPrefix(environmentType)).toBe(prefix); + expect(obfuscateApiKey(environmentType, root.lastFour)).toBe( + `${prefix}••••••••${root.lastFour}` + ); + expect(obfuscateApiKey(environmentType, additional.lastFour, "additional")).toBe( + `${prefix}sk_••••••••${additional.lastFour}` + ); + }); + + test("generates unique keys", () => { + const keys = new Set( + Array.from({ length: 100 }, () => generateAdditionalApiKey("PRODUCTION").apiKey) + ); + + expect(keys.size).toBe(100); + }); +}); diff --git a/apps/webapp/app/utils/apiKeys.ts b/apps/webapp/app/utils/apiKeys.ts new file mode 100644 index 00000000000..556c203a49d --- /dev/null +++ b/apps/webapp/app/utils/apiKeys.ts @@ -0,0 +1,51 @@ +import { createHash } from "node:crypto"; +import type { RuntimeEnvironmentType } from "@trigger.dev/database"; +import { customAlphabet } from "nanoid"; + +const apiKeyId = customAlphabet( + "1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ", + 24 +); + +export function hashApiKey(apiKey: string): string { + return createHash("sha256").update(apiKey, "utf8").digest("hex"); +} + +function generatedApiKey(apiKey: string) { + return { + apiKey, + keyHash: hashApiKey(apiKey), + lastFour: apiKey.slice(-4), + }; +} + +export function generateRootApiKey(environmentType: RuntimeEnvironmentType) { + // Root keys intentionally use the same 24-character entropy as additional keys. + return generatedApiKey(`${apiKeyPrefix(environmentType)}${apiKeyId()}`); +} + +export function generateAdditionalApiKey(environmentType: RuntimeEnvironmentType) { + return generatedApiKey(`${apiKeyPrefix(environmentType)}sk_${apiKeyId()}`); +} + +export function apiKeyPrefix(environmentType: RuntimeEnvironmentType): string { + switch (environmentType) { + case "DEVELOPMENT": + return "tr_dev_"; + case "STAGING": + return "tr_stg_"; + case "PRODUCTION": + return "tr_prod_"; + case "PREVIEW": + return "tr_preview_"; + } +} + +export function obfuscateApiKey( + environmentType: RuntimeEnvironmentType, + lastFour: string, + kind: "root" | "additional" = "root" +): string { + const discriminator = kind === "additional" ? "sk_" : ""; + return `${apiKeyPrefix(environmentType)}${discriminator}••••••••${lastFour}`; +} diff --git a/internal-packages/database/prisma/migrations/20260723112558_add_environment_api_keys/migration.sql b/internal-packages/database/prisma/migrations/20260723112558_add_environment_api_keys/migration.sql new file mode 100644 index 00000000000..63957b702df --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260723112558_add_environment_api_keys/migration.sql @@ -0,0 +1,30 @@ +-- CreateTable +CREATE TABLE "public"."api_keys" ( + "id" TEXT NOT NULL, + "name" TEXT NOT NULL, + "key_hash" TEXT NOT NULL, + "last_four" TEXT NOT NULL, + "runtime_environment_id" TEXT NOT NULL, + "created_by_user_id" TEXT, + "preset_id" TEXT, + "scopes" TEXT[] NOT NULL, + "last_used_at" TIMESTAMP(3), + "revoked_at" TIMESTAMP(3), + "expires_at" TIMESTAMP(3), + "updated_at" TIMESTAMP(3) NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "api_keys_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "api_keys_key_hash_key" ON "public"."api_keys"("key_hash"); + +-- CreateIndex +CREATE INDEX "api_keys_runtime_environment_id_revoked_at_created_at_idx" ON "public"."api_keys"("runtime_environment_id", "revoked_at", "created_at" DESC); + +-- AddForeignKey +ALTER TABLE "public"."api_keys" ADD CONSTRAINT "api_keys_runtime_environment_id_fkey" FOREIGN KEY ("runtime_environment_id") REFERENCES "public"."RuntimeEnvironment"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "public"."api_keys" ADD CONSTRAINT "api_keys_created_by_user_id_fkey" FOREIGN KEY ("created_by_user_id") REFERENCES "public"."User"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 2998df20937..74c6d72a9c5 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -69,6 +69,7 @@ model User { invitationCode InvitationCode? @relation(fields: [invitationCodeId], references: [id]) invitationCodeId String? personalAccessTokens PersonalAccessToken[] + createdApiKeys ApiKey[] deployments WorkerDeployment[] backupCodes MfaBackupCode[] bulkActions BulkActionGroup[] @@ -392,6 +393,7 @@ model RuntimeEnvironment { playgroundConversations PlaygroundConversation[] errorGroupStates ErrorGroupState[] taskIdentifiers TaskIdentifier[] + apiKeys ApiKey[] revokedApiKeys RevokedApiKey[] // A partial unique index also enforces one STAGING/PREVIEW root per project and type. @@ -403,6 +405,31 @@ model RuntimeEnvironment { @@index([organizationId]) } +model ApiKey { + id String @id @default(cuid()) + name String + keyHash String @unique @map("key_hash") + lastFour String @map("last_four") + + runtimeEnvironment RuntimeEnvironment @relation(fields: [runtimeEnvironmentId], references: [id], onDelete: Cascade, onUpdate: Cascade) + runtimeEnvironmentId String @map("runtime_environment_id") + + createdBy User? @relation(fields: [createdByUserId], references: [id], onDelete: SetNull, onUpdate: Cascade) + createdByUserId String? @map("created_by_user_id") + + presetId String? @map("preset_id") + scopes String[] + + lastUsedAt DateTime? @map("last_used_at") + revokedAt DateTime? @map("revoked_at") + expiresAt DateTime? @map("expires_at") + updatedAt DateTime @updatedAt @map("updated_at") + createdAt DateTime @default(now()) @map("created_at") + + @@index([runtimeEnvironmentId, revokedAt, createdAt(sort: Desc)]) + @@map("api_keys") +} + /// Records of previously-valid API keys that are still accepted for authentication /// during a grace window after rotation. Extend or end the grace period by updating `expiresAt`. model RevokedApiKey { diff --git a/internal-packages/rbac/src/ability.test.ts b/internal-packages/rbac/src/ability.test.ts index a3250bd0030..3497ba66690 100644 --- a/internal-packages/rbac/src/ability.test.ts +++ b/internal-packages/rbac/src/ability.test.ts @@ -5,6 +5,7 @@ import { denyAbility, buildFallbackAbility, buildJwtAbility, + scopesWithinAbility, } from "./ability.js"; describe("permissiveAbility", () => { @@ -123,6 +124,43 @@ describe("buildJwtAbility", () => { }); }); +describe("scopesWithinAbility", () => { + it("allows subsets and preserves ids containing colons", () => { + const result = scopesWithinAbility( + ["read:runs:run_abc", "read:tags:env:staging"], + buildJwtAbility(["read:runs", "read:tags:env:staging"]) + ); + + expect(result).toEqual({ ok: true, deniedScopes: [] }); + }); + + it("rejects scopes that broaden or exceed the ability", () => { + const result = scopesWithinAbility( + ["trigger:tasks:send-email", "trigger:tasks", "read:runs"], + buildJwtAbility(["trigger:tasks:send-email"]) + ); + + expect(result).toEqual({ + ok: false, + deniedScopes: ["trigger:tasks", "read:runs"], + }); + }); + + it("allows arbitrary valid scopes for a permissive ability", () => { + expect(scopesWithinAbility(["read:runs", "admin"], permissiveAbility)).toEqual({ + ok: true, + deniedScopes: [], + }); + }); + + it("rejects malformed scopes for restricted abilities", () => { + expect(scopesWithinAbility(["read"], buildJwtAbility(["read:all"]))).toEqual({ + ok: false, + deniedScopes: ["read"], + }); + }); +}); + describe("buildJwtAbility — array resources", () => { it("authorizes when any resource in the array passes a scope check", () => { const ability = buildJwtAbility(["read:batch:batch_abc"]); diff --git a/internal-packages/rbac/src/ability.ts b/internal-packages/rbac/src/ability.ts index 6cf22d7643e..9cd5cab68e7 100644 --- a/internal-packages/rbac/src/ability.ts +++ b/internal-packages/rbac/src/ability.ts @@ -4,7 +4,7 @@ import type { RbacAbility } from "@trigger.dev/plugins"; // @trigger.dev/plugins so a public token decodes identically whoever // serves the request. Re-exported here so existing importers keep their // `./ability.js` import. -export { buildJwtAbility } from "@trigger.dev/plugins"; +export { buildJwtAbility, scopesWithinAbility } from "@trigger.dev/plugins"; /** Every authenticated non-admin subject: can do anything, cannot do super-user actions. */ export const permissiveAbility: RbacAbility = { diff --git a/internal-packages/rbac/src/apiKeyPolicyDefaults.test.ts b/internal-packages/rbac/src/apiKeyPolicyDefaults.test.ts new file mode 100644 index 00000000000..c023bea1c3f --- /dev/null +++ b/internal-packages/rbac/src/apiKeyPolicyDefaults.test.ts @@ -0,0 +1,58 @@ +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, it, vi } from "vitest"; + +// The API-key policy methods are OPTIONAL on RoleBaseAccessController so a +// plugin compiled against an older OSS commit still satisfies the contract +// (the plugin is built against whichever OSS source its base image carries). +// LazyController is what turns that partial surface into a total one, and these +// tests pin the defaults it substitutes — in particular that a missing +// prepareApiKeyPolicy fails CLOSED rather than resolving to full access. +// +// A stand-in for the cloud plugin, which isn't installed in this repo. The +// factory supplies the specifier, so no real module has to resolve. +vi.mock("@triggerdotdev/plugins/rbac", () => ({ + default: { + create: () => ({ + // Deliberately omits apiKeyPresets / prepareApiKeyPolicy / + // describeApiKeyPolicy — this is a pre-contract plugin. + isUsingPlugin: async () => true, + }), + }, +})); + +const prismaPlaceholder = {} as unknown as PrismaClient; + +describe("LazyController API-key policy defaults (plugin predates the contract)", () => { + async function controller() { + const loader = (await import("./index.js")).default; + const instance = loader.create(prismaPlaceholder); + // Guard against a silent fallback: if the mock didn't take, these + // assertions would be checking the fallback's real implementations. + await expect(instance.isUsingPlugin()).resolves.toBe(true); + return instance; + } + + it("reports no preset catalogue rather than throwing", async () => { + await expect((await controller()).apiKeyPresets("org_123")).resolves.toBeNull(); + }); + + it("refuses to prepare a policy — including FULL_ACCESS", async () => { + const result = await ( + await controller() + ).prepareApiKeyPolicy({ + organizationId: "org_123", + presetId: "FULL_ACCESS", + }); + + // The critical assertion: absence must never resolve to `{ ok: true }` with + // an admin scope. A plugin below the contract cannot mint any credential. + expect(result.ok).toBe(false); + expect(result).not.toHaveProperty("policy"); + }); + + it("describes a policy as having nothing extra to show", async () => { + await expect( + (await controller()).describeApiKeyPolicy({ presetId: "TRIGGER_ONLY", scopes: ["read:runs"] }) + ).resolves.toEqual({}); + }); +}); diff --git a/internal-packages/rbac/src/fallback.ts b/internal-packages/rbac/src/fallback.ts index 7ada941f755..4cf1e765c7a 100644 --- a/internal-packages/rbac/src/fallback.ts +++ b/internal-packages/rbac/src/fallback.ts @@ -13,7 +13,11 @@ import type { RoleMutationResult, UserActorAuthResult, } from "@trigger.dev/plugins"; -import { isUserActorToken, verifyUserActorToken } from "@trigger.dev/plugins"; +import { + FULL_ACCESS_PRESET_ID, + isUserActorToken, + verifyUserActorToken, +} from "@trigger.dev/plugins"; import { createHash } from "node:crypto"; import type { PrismaClient } from "@trigger.dev/database"; import { validateJWT } from "@trigger.dev/core/v3/jwt"; @@ -374,6 +378,34 @@ class RoleBaseAccessFallbackController implements RoleBaseAccessController { return null; } + async apiKeyPresets(_organizationId: string) { + return null; + } + + async prepareApiKeyPolicy(params: { + organizationId: string; + presetId: string; + taskIdentifiers?: string[]; + }) { + // Without a plugin there is no preset catalogue, so full access is the only + // policy on offer, but the caller still has to ask for it by name. Any + // other preset, or any task selection, is a restricted key and unavailable. + if (params.presetId !== FULL_ACCESS_PRESET_ID || (params.taskIdentifiers?.length ?? 0) > 0) { + return { ok: false as const, error: "API key access presets are not available" }; + } + + // `presetId: null` because this install has no catalogue to reference. The + // persisted scopes remain the source of truth for authorization. + return { + ok: true as const, + policy: { presetId: null, scopes: ["admin"] }, + }; + } + + async describeApiKeyPolicy() { + return {}; + } + async allPermissions(): Promise<Permission[]> { return []; } diff --git a/internal-packages/rbac/src/index.ts b/internal-packages/rbac/src/index.ts index eb34da4f0a1..6873c9d479f 100644 --- a/internal-packages/rbac/src/index.ts +++ b/internal-packages/rbac/src/index.ts @@ -1,5 +1,8 @@ import type { + ApiKeyPolicyDescription, + ApiKeyPreset, Permission, + PrepareApiKeyPolicyResult, RbacAbility, RbacDatabaseConfig, Role, @@ -12,7 +15,21 @@ import type { import type { PrismaClient } from "@trigger.dev/database"; import { RoleBaseAccessFallback } from "./fallback.js"; export type { RoleBaseAccessController, RbacAbility, RbacResource } from "@trigger.dev/plugins"; + +/** + * The controller surface as the HOST sees it, after LazyController has filled in + * defaults for the optional capability methods a plugin may not implement. + * + * `RoleBaseAccessController` is the *plugin-facing* contract, where capability + * extensions are optional so an older plugin still satisfies it. Host code + * always talks to the LazyController singleton (`rbac`), which never omits a + * method — so host consumers should depend on this type, not on the plugin + * contract, and get a total surface without writing their own guards. + */ +export type HostRbacController = Required<RoleBaseAccessController>; export type { UserActorAuthResult, UserActorClaims } from "@trigger.dev/plugins"; +export { buildJwtAbility, scopesWithinAbility } from "./ability.js"; +export { FULL_ACCESS_PRESET_ID, scopesGrantFullAccess } from "@trigger.dev/plugins"; // Re-export the user-actor token grammar so the webapp mints/checks tokens // through @trigger.dev/rbac (it doesn't import @trigger.dev/plugins directly). export { @@ -213,6 +230,41 @@ class LazyController implements RoleBaseAccessController { return (await this.c()).systemRoles(...args); } + // The API-key policy methods are optional on the controller contract (see the + // note on RoleBaseAccessController) so a plugin compiled against an older OSS + // commit still satisfies it. LazyController is where that optional surface is + // normalized into a total one: every host caller goes through `rbac`, so the + // absent-plugin default lives here once instead of at each call site. + async apiKeyPresets( + ...args: Parameters<NonNullable<RoleBaseAccessController["apiKeyPresets"]>> + ): Promise<ApiKeyPreset[] | null> { + const controller = await this.c(); + // Same meaning as the no-plugin fallback: no catalogue to offer. + return controller.apiKeyPresets ? controller.apiKeyPresets(...args) : null; + } + + async prepareApiKeyPolicy( + ...args: Parameters<NonNullable<RoleBaseAccessController["prepareApiKeyPolicy"]>> + ): Promise<PrepareApiKeyPolicyResult> { + const controller = await this.c(); + if (!controller.prepareApiKeyPolicy) { + // Fail closed. A plugin that predates this contract must not be able to + // mint a credential — least of all a full-access one — so creation stops + // outright. Keys already issued are unaffected: they authorize from the + // scopes persisted on their row, compiled by the host bearer resolver. + return { ok: false, error: "API key access presets are not available" }; + } + return controller.prepareApiKeyPolicy(...args); + } + + async describeApiKeyPolicy( + ...args: Parameters<NonNullable<RoleBaseAccessController["describeApiKeyPolicy"]>> + ): Promise<ApiKeyPolicyDescription> { + const controller = await this.c(); + // Presentation only — an undescribed policy renders from its stored scopes. + return controller.describeApiKeyPolicy ? controller.describeApiKeyPolicy(...args) : {}; + } + async allPermissions( ...args: Parameters<RoleBaseAccessController["allPermissions"]> ): Promise<Permission[]> { @@ -293,7 +345,13 @@ class LazyController implements RoleBaseAccessController { class RoleBaseAccess { // Synchronous — returns a lazy controller that resolves any installed // plugin on first call. - create(prisma: RbacPrismaInput, options?: RbacCreateOptions): RoleBaseAccessController { + // + // Returns HostRbacController, not RoleBaseAccessController: the latter is the + // plugin-facing contract whose capability methods are optional, and + // LazyController has already substituted defaults for any the installed plugin + // omits. Handing back the total surface is what keeps host callers from having + // to guard (or, worse, from inventing their own absent-plugin default). + create(prisma: RbacPrismaInput, options?: RbacCreateOptions): HostRbacController { return new LazyController(prisma, options); } } diff --git a/packages/core/src/v3/jwt.ts b/packages/core/src/v3/jwt.ts index 6845225be8a..b1dec457ada 100644 --- a/packages/core/src/v3/jwt.ts +++ b/packages/core/src/v3/jwt.ts @@ -14,6 +14,40 @@ export const JWT_ALGORITHM = "HS256"; export const JWT_ISSUER = "https://id.trigger.dev"; export const JWT_AUDIENCE = "https://api.trigger.dev"; +function decodeJWTPayload(token: string): unknown { + const parts = token.split("."); + const encodedPayload = parts[1]; + if (parts.length !== 3 || !encodedPayload) return; + + try { + const base64 = encodedPayload + .replace(/-/g, "+") + .replace(/_/g, "/") + .padEnd(Math.ceil(encodedPayload.length / 4) * 4, "="); + const bytes = Uint8Array.from(atob(base64), (character) => character.charCodeAt(0)); + return JSON.parse(new TextDecoder().decode(bytes)); + } catch { + return; + } +} + +export function isPublicJWT(token: string): boolean { + const payload = decodeJWTPayload(token); + return ( + payload !== null && typeof payload === "object" && "pub" in payload && payload.pub === true + ); +} + +export function extractJWTSub(token: string): string | undefined { + const payload = decodeJWTPayload(token); + return payload !== null && + typeof payload === "object" && + "sub" in payload && + typeof payload.sub === "string" + ? payload.sub + : undefined; +} + export async function generateJWT(options: GenerateJWTOptions): Promise<string> { const { SignJWT } = await import("jose"); diff --git a/packages/plugins/src/index.ts b/packages/plugins/src/index.ts index 1c561d7452a..198efce3fc3 100644 --- a/packages/plugins/src/index.ts +++ b/packages/plugins/src/index.ts @@ -18,10 +18,22 @@ export type { RbacPluginConfig, RbacDatabaseConfig, SystemRole, + ApiKeyPreset, + ApiKeyPolicy, + PrepareApiKeyPolicyResult, + ApiKeyPolicyDescription, AuthenticatedEnvironment, + RbacScopeAction, + RbacScopeResourceType, } from "./rbac.js"; -export { buildJwtAbility } from "./rbac.js"; +export { + buildJwtAbility, + buildScope, + FULL_ACCESS_PRESET_ID, + scopesGrantFullAccess, + scopesWithinAbility, +} from "./rbac.js"; export { isUserActorToken, signUserActorToken, diff --git a/packages/plugins/src/rbac.ts b/packages/plugins/src/rbac.ts index b31abd1215b..52c6da8fdc9 100644 --- a/packages/plugins/src/rbac.ts +++ b/packages/plugins/src/rbac.ts @@ -19,6 +19,27 @@ export type SystemRole = { available: boolean; }; +export type ApiKeyPreset = { + id: string; + label: string; + description: string; + usesTaskSelection: boolean; + available: boolean; +}; + +export type ApiKeyPolicy = { + presetId: string | null; + scopes: string[]; +}; + +export type PrepareApiKeyPolicyResult = + | { ok: true; policy: ApiKeyPolicy } + | { ok: false; error: string }; + +export type ApiKeyPolicyDescription = { + taskIdentifiers?: string[]; +}; + export type Permission = { // `<action>:<subject>` — display name, derived from the ability rule. name: string; @@ -47,6 +68,18 @@ export type RbacSubject = | { type: "user"; userId: string; organizationId: string; projectId?: string } | { type: "personalAccessToken"; tokenId: string; organizationId: string; projectId?: string } | { type: "publicJWT"; environmentId: string; organizationId: string; projectId?: string } + // A host-owned additional environment API key. The host builds its ability + // from the effective scopes stored with the credential. Route builders use + // `restricted` to fail closed when a scoped key reaches an endpoint without a + // declared authorization resource. `apiKeyId` identifies the key for + // attribution. Root/legacy environment keys keep the `user` subject. + | { + type: "apiKey"; + apiKeyId: string; + restricted: boolean; + organizationId: string; + projectId?: string; + } // Delegated user-actor token (`tr_uat_…`): a short-lived, stateless // credential that authenticates as `userId`. `client` records what minted // it (e.g. a dashboard agent) for attribution. @@ -110,29 +143,91 @@ export interface RbacAbility { * which auth path serves the request — two copies of this grammar would * drift, and the difference would silently change what a token grants. */ +function parseScope(scope: string): { action: string; type?: string; id?: string } | undefined { + // Only the first two colons are delimiters — everything after the + // second colon is the resource id (which may itself contain colons, + // e.g. user-provided tags like "env:staging"). Naive + // `split(":")` + 3-tuple destructuring truncates such ids. + const parts = scope.split(":"); + const action = parts[0]; + if (!action) return undefined; + + return { + action, + type: parts[1] || undefined, + id: parts.length > 2 ? parts.slice(2).join(":") || undefined : undefined, + }; +} + +/** Only exact bare `admin` represents unrestricted access. */ +export function scopesGrantFullAccess(scopes: readonly string[]): boolean { + return scopes.includes("admin"); +} + +/** + * The one preset every install supports, including those with no preset + * catalogue at all. `prepareApiKeyPolicy` takes a required `presetId`, so + * callers that want a root-key-equivalent credential name this rather than + * relying on a default — see the note on that method. + */ +export const FULL_ACCESS_PRESET_ID = "FULL_ACCESS"; + +// The closed vocabulary a public token / API-key scope can address. This is +// the ONE place the `action:type[:id]` string grammar's terms are enumerated, +// shared by the scope *generator* (a plugin's preset builder) and the host's +// scope *checks* so the two cannot drift on a rename/typo. Every value here is +// already public: it appears in the OSS webapp's route authorization +// declarations and in `buildJwtAbility`'s grammar. A plugin that gates +// resources beyond this set can widen the union locally — do not add +// non-public terms here. +export type RbacScopeAction = "read" | "write" | "trigger" | "batchTrigger"; + +export type RbacScopeResourceType = + | "runs" + | "tasks" + | "batch" + | "queues" + | "deployments" + | "envvars" + | "apiKeys" + | "sessions" + | "waitpoints" + | "tags" + | "query"; + +/** + * Builds a single `action:type[:id]` scope string from typed parts. Scope + * generators should construct every scope through this helper so the shared + * `RbacScopeAction` / `RbacScopeResourceType` unions catch a mistyped or + * renamed term at compile time — the drift the grammar comment above warns + * about. `id` may itself contain colons (e.g. a tag like `env:staging`); it is + * appended verbatim and decoded by `parseScope`'s slice-join. + */ +export function buildScope( + action: RbacScopeAction, + type: RbacScopeResourceType, + id?: string +): string { + return id ? `${action}:${type}:${id}` : `${action}:${type}`; +} + export function buildJwtAbility(scopes: string[]): RbacAbility { const matches = (action: string, r: RbacResource): boolean => scopes.some((scope) => { - // Only the first two colons are delimiters — everything after the - // second colon is the resource id (which may itself contain colons, - // e.g. user-provided tags like "env:staging"). Naive - // `split(":")` + 3-tuple destructuring truncated such ids to the - // first segment and silently failed to match. - const parts = scope.split(":"); - const scopeAction = parts[0]; - const scopeType = parts[1]; - const scopeId = parts.length > 2 ? parts.slice(2).join(":") : undefined; + const parsed = parseScope(scope); + if (!parsed) return false; + // Bare `admin` is the universal wildcard. `admin:<type>` is *not* — // it falls through to normal matching as action="admin" against // resources of that type. Treating `admin:<anything>` as universal // would silently broaden any such tokens beyond the narrow, // route-listed grant they had before scope-based abilities. - if (scopeAction === "admin" && !scopeType) return true; - if (scopeAction !== action && scopeAction !== "*") return false; - if (scopeType === "all") return true; - if (scopeType !== r.type) return false; - if (!scopeId) return true; - return scopeId === r.id; + if (parsed.action === "admin" && !parsed.type) return true; + if (parsed.action !== action && parsed.action !== "*") return false; + if (parsed.type === "all") return true; + if (parsed.type !== r.type) return false; + if (!parsed.id) return true; + return parsed.id === r.id; }); return { can(action: string, resource: RbacResource | RbacResource[]): boolean { @@ -148,6 +243,30 @@ export function buildJwtAbility(scopes: string[]): RbacAbility { }; } +/** + * Checks requested public-token scopes against an already-authenticated ability. + * Scope parsing intentionally lives beside buildJwtAbility so minting and + * validation use the same action:type[:id] grammar. + */ +export function scopesWithinAbility( + scopes: string[], + ability: RbacAbility +): { ok: boolean; deniedScopes: string[] } { + const deniedScopes = scopes.filter((scope) => { + const parsed = parseScope(scope); + if (!parsed || (!parsed.type && parsed.action !== "admin")) { + return true; + } + + return !ability.can(parsed.action, { + type: parsed.type ?? "all", + ...(parsed.id ? { id: parsed.id } : {}), + }); + }); + + return { ok: deniedScopes.length === 0, deniedScopes }; +} + // ── Delegated user-actor token grammar ─────────────────────────────────── // // A `tr_uat_…` token is the JWT body (signed HS256 with the platform secret) @@ -348,6 +467,41 @@ export interface RoleBaseAccessController { // an upgrade badge or hide them. systemRoles(organizationId: string): Promise<SystemRole[] | null>; + // Plugin-owned API-key policy catalogue and policy generation. A null + // catalogue means no plugin is installed. The host owns credentials and + // persists the effective policy returned by prepareApiKeyPolicy(). + // + // These three are OPTIONAL, and are the first optional members of this + // interface — the distinction is deliberate. Methods the host cannot serve a + // request without (authenticateBearer, authenticatePat, authenticateSession) + // stay required. Capability extensions the host can degrade past are + // optional, so a plugin built against an older contract still satisfies this + // interface. That matters because the plugin is compiled against whichever + // OSS commit its base image carries: making an additive capability required + // turns every such addition into a lockstep two-repo merge, and turns a + // plugin rollback into an image build failure instead of a graceful + // degradation. + // + // Absence must fail CLOSED, and each default is chosen so it does: + // - apiKeyPresets -> null (identical to "no plugin installed") + // - prepareApiKeyPolicy -> { ok: false } — NEVER a full-access default; + // key creation stops, while already-issued keys + // keep authorizing from their persisted scopes + // - describeApiKeyPolicy -> {} (presentation only) + // LazyController applies these defaults, so host callers going through + // `rbac` see a total surface and cannot forget the guard. + apiKeyPresets?(organizationId: string): Promise<ApiKeyPreset[] | null>; + // `presetId` is deliberately required: an authorization function must not + // have an implicit default, least of all a full-access one. A caller that + // omits the field should fail to compile rather than silently mint an admin + // credential. Installs with no preset catalogue pass "FULL_ACCESS". + prepareApiKeyPolicy?(params: { + organizationId: string; + presetId: string; + taskIdentifiers?: string[]; + }): Promise<PrepareApiKeyPolicyResult>; + describeApiKeyPolicy?(policy: ApiKeyPolicy): Promise<ApiKeyPolicyDescription>; + // Role introspection. The fallback returns []; a plugin may return // its own role catalogue. allPermissions(organizationId: string): Promise<Permission[]>; From 878c15811aca8339a751aa0c2211012db7a47ce5 Mon Sep 17 00:00:00 2001 From: Chris Arderne <chris@trigger.dev> Date: Wed, 29 Jul 2026 17:36:25 +0100 Subject: [PATCH 33/67] fix(cli): redact environment values from build debug logs (#4420) --- .changeset/silent-build-logs.md | 5 +++ packages/cli-v3/src/build/buildWorker.ts | 7 ++-- .../src/build/buildWorkerLogging.test.ts | 35 +++++++++++++++++++ .../cli-v3/src/build/buildWorkerLogging.ts | 21 +++++++++++ 4 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 .changeset/silent-build-logs.md create mode 100644 packages/cli-v3/src/build/buildWorkerLogging.test.ts create mode 100644 packages/cli-v3/src/build/buildWorkerLogging.ts diff --git a/.changeset/silent-build-logs.md b/.changeset/silent-build-logs.md new file mode 100644 index 00000000000..17c3b9ab0f3 --- /dev/null +++ b/.changeset/silent-build-logs.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Prevent build debug logs from including environment variable values. diff --git a/packages/cli-v3/src/build/buildWorker.ts b/packages/cli-v3/src/build/buildWorker.ts index c3e1641ade1..fc199142492 100644 --- a/packages/cli-v3/src/build/buildWorker.ts +++ b/packages/cli-v3/src/build/buildWorker.ts @@ -15,11 +15,12 @@ import { join, relative, sep } from "node:path"; import { generateContainerfile } from "../deploy/buildImage.js"; import { writeFile } from "node:fs/promises"; import { buildManifestToJSON } from "../utilities/buildManifest.js"; +import { logger } from "../utilities/logger.js"; import { readPackageJSON } from "pkg-types"; import { writeJSONFile } from "../utilities/fileSystem.js"; import { isWindows } from "std-env"; import { pathToFileURL } from "node:url"; -import { logger } from "../utilities/logger.js"; +import { logBuildWorkerStart } from "./buildWorkerLogging.js"; import { SdkVersionExtractor } from "./plugins.js"; import { spinner } from "../utilities/windows.js"; @@ -42,9 +43,7 @@ export type BuildWorkerOptions = { }; export async function buildWorker(options: BuildWorkerOptions) { - logger.debug("Starting buildWorker", { - options, - }); + logBuildWorkerStart(options); const resolvedConfig = options.resolvedConfig; diff --git a/packages/cli-v3/src/build/buildWorkerLogging.test.ts b/packages/cli-v3/src/build/buildWorkerLogging.test.ts new file mode 100644 index 00000000000..8b5f5827f5f --- /dev/null +++ b/packages/cli-v3/src/build/buildWorkerLogging.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it, vi } from "vitest"; +import { logBuildWorkerStart } from "./buildWorkerLogging.js"; +import { logger } from "../utilities/logger.js"; + +const secret = "build-worker-secret-value"; + +describe("logBuildWorkerStart", () => { + it.each(["deploy", "unmanaged"] as const)( + "does not log environment values for %s builds", + (target) => { + const debug = vi.spyOn(logger, "debug").mockImplementation(() => {}); + + logBuildWorkerStart({ + target, + branch: "main", + envVars: { BUILD_SECRET: secret }, + rewritePaths: true, + forcedExternals: ["example-package"], + }); + + expect(debug).toHaveBeenCalledOnce(); + expect(JSON.stringify(debug.mock.calls)).not.toContain(secret); + expect(debug).toHaveBeenCalledWith("Starting buildWorker", { + target, + hasBranch: true, + envVarCount: 1, + rewritePaths: true, + forcedExternalsCount: 1, + plain: false, + }); + + debug.mockRestore(); + } + ); +}); diff --git a/packages/cli-v3/src/build/buildWorkerLogging.ts b/packages/cli-v3/src/build/buildWorkerLogging.ts new file mode 100644 index 00000000000..867c8df31a2 --- /dev/null +++ b/packages/cli-v3/src/build/buildWorkerLogging.ts @@ -0,0 +1,21 @@ +import { logger } from "../utilities/logger.js"; + +type BuildWorkerLogOptions = { + target: string; + branch?: string; + envVars?: Record<string, string>; + rewritePaths?: boolean; + forcedExternals?: string[]; + plain?: boolean; +}; + +export function logBuildWorkerStart(options: BuildWorkerLogOptions) { + logger.debug("Starting buildWorker", { + target: options.target, + hasBranch: options.branch !== undefined, + envVarCount: Object.keys(options.envVars ?? {}).length, + rewritePaths: options.rewritePaths ?? false, + forcedExternalsCount: options.forcedExternals?.length ?? 0, + plain: options.plain ?? false, + }); +} From ed8f5e1297b0610ecb0da274f3752e43ab7e63af Mon Sep 17 00:00:00 2001 From: Chris Arderne <chris@trigger.dev> Date: Wed, 29 Jul 2026 17:37:58 +0100 Subject: [PATCH 34/67] fix(webapp): stop logging every environment on a lookup miss (#4402) --- apps/webapp/app/presenters/OrganizationsPresenter.server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/webapp/app/presenters/OrganizationsPresenter.server.ts b/apps/webapp/app/presenters/OrganizationsPresenter.server.ts index 25a8ed22d7f..c002be691b9 100644 --- a/apps/webapp/app/presenters/OrganizationsPresenter.server.ts +++ b/apps/webapp/app/presenters/OrganizationsPresenter.server.ts @@ -235,7 +235,7 @@ export class OrganizationsPresenter { if (!env) { logger.info("Not Found: environment", { environmentSlug, - environments, + environmentCount: environments.length, }); } } From a09817169f006e3c0560c62ca181b8f242abbd34 Mon Sep 17 00:00:00 2001 From: Chris Arderne <chris@trigger.dev> Date: Wed, 29 Jul 2026 17:59:27 +0100 Subject: [PATCH 35/67] fix(webapp): stop logging full batch item contents in batchTriggerV3 (#4404) --- apps/webapp/app/v3/services/batchTriggerV3.server.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/apps/webapp/app/v3/services/batchTriggerV3.server.ts b/apps/webapp/app/v3/services/batchTriggerV3.server.ts index fdb87d2d2ca..563ef446bcc 100644 --- a/apps/webapp/app/v3/services/batchTriggerV3.server.ts +++ b/apps/webapp/app/v3/services/batchTriggerV3.server.ts @@ -409,10 +409,6 @@ export class BatchTriggerV3Service extends BaseService { {} as Record<string, typeof body.items> ); - logger.debug("[BatchTriggerV2][call] Grouped items by task identifier", { - itemsByTask, - }); - const idempotencyKeyLookups = Object.entries(itemsByTask).flatMap(([taskIdentifier, items]) => { const idempotencyKeys = Array.from( new Set( From 8ebc8a41af667577882723af6e44ae02252a2eed Mon Sep 17 00:00:00 2001 From: Chris Arderne <chris@trigger.dev> Date: Wed, 29 Jul 2026 17:59:36 +0100 Subject: [PATCH 36/67] fix(webapp,redis-worker): stop logging raw metadata, alert payloads, and job items (#4403) --- apps/webapp/app/services/logger.server.ts | 2 +- .../metadata/updateMetadata.server.ts | 29 +++++++---- apps/webapp/app/utils/packets.ts | 10 +++- .../v3/services/alerts/deliverAlert.server.ts | 50 +++++++++++++++---- packages/redis-worker/src/worker.ts | 10 ++-- 5 files changed, 74 insertions(+), 27 deletions(-) diff --git a/apps/webapp/app/services/logger.server.ts b/apps/webapp/app/services/logger.server.ts index 15b248f0d94..b03802afefd 100644 --- a/apps/webapp/app/services/logger.server.ts +++ b/apps/webapp/app/services/logger.server.ts @@ -50,7 +50,7 @@ function flattenArgs(args: Array<Record<string, unknown> | undefined>) { export const logger = new Logger( "webapp", (process.env.APP_LOG_LEVEL ?? "info") as LogLevel, - ["examples", "output", "connectionString", "payload"], + ["examples", "output", "connectionString", "payload", "metadata", "seedMetadata"], sensitiveDataReplacer, () => { const fields = currentFieldsStore.getStore(); diff --git a/apps/webapp/app/services/metadata/updateMetadata.server.ts b/apps/webapp/app/services/metadata/updateMetadata.server.ts index c44dacf65c5..07dc236c976 100644 --- a/apps/webapp/app/services/metadata/updateMetadata.server.ts +++ b/apps/webapp/app/services/metadata/updateMetadata.server.ts @@ -6,7 +6,11 @@ import type { import { applyMetadataOperations, parsePacket } from "@trigger.dev/core/v3"; import type { PrismaClientOrTransaction } from "~/db.server"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; -import { handleMetadataPacket, MetadataTooLargeError } from "~/utils/packets"; +import { + handleMetadataPacket, + handleMetadataPacketWithByteLength, + MetadataTooLargeError, +} from "~/utils/packets"; import { ServiceValidationError } from "~/v3/services/common.server"; import { Effect, Schedule, Duration, Fiber } from "effect"; import { type RuntimeFiber } from "effect/Fiber"; @@ -91,9 +95,14 @@ export class UpdateMetadataService { this._bufferedOperations.clear(); yield* Effect.sync(() => { - if (this.flushLoggingEnabled) { + if (this.flushLoggingEnabled && currentOperations.size > 0) { + const operationCount = Array.from(currentOperations.values()).reduce( + (sum, ops) => sum + ops.length, + 0 + ); this.logger.debug(`[UpdateMetadataService] Flushing operations`, { - operations: Object.fromEntries(currentOperations), + runCount: currentOperations.size, + operationCount, }); } }); @@ -520,9 +529,9 @@ export class UpdateMetadataService { if (this.flushLoggingEnabled) { this.logger.debug(`[updateRunMetadataWithOperations] Updated metadata for run`, { - metadata: applyResults.newMetadata, - operations: operations, runId, + metadataKeyCount: Object.keys(applyResults.newMetadata).length, + operationCount: operations.length, }); } @@ -549,16 +558,18 @@ export class UpdateMetadataService { body: UpdateMetadataRequestBody, existingMetadata: IOPacket ): Promise<{ metadata: Record<string, unknown> | undefined; updatedAtMs?: number }> { - const metadataPacket = handleMetadataPacket( + const metadataPacketWithByteLength = handleMetadataPacketWithByteLength( body.metadata, "application/json", this.maximumSize ); - if (!metadataPacket) { + if (!metadataPacketWithByteLength) { return { metadata: {} }; } + const { packet: metadataPacket, byteLength: metadataSizeBytes } = metadataPacketWithByteLength; + let updatedAtMs: number | undefined; if ( @@ -567,8 +578,8 @@ export class UpdateMetadataService { ) { if (this.flushLoggingEnabled) { this.logger.debug(`[updateRunMetadataDirectly] Updating metadata directly for run`, { - metadata: metadataPacket.data, runId, + metadataSizeBytes, }); } @@ -607,7 +618,7 @@ export class UpdateMetadataService { if (this.flushLoggingEnabled) { this.logger.debug(`[ingestRunOperations] Ingesting operations for run`, { runId, - bufferedOperations, + operationCount: bufferedOperations.length, }); } diff --git a/apps/webapp/app/utils/packets.ts b/apps/webapp/app/utils/packets.ts index 7a522d6f7af..27217c7f2ae 100644 --- a/apps/webapp/app/utils/packets.ts +++ b/apps/webapp/app/utils/packets.ts @@ -13,6 +13,14 @@ export function handleMetadataPacket( metadataType: string, maximumSize: number ): IOPacket | undefined { + return handleMetadataPacketWithByteLength(metadata, metadataType, maximumSize)?.packet; +} + +export function handleMetadataPacketWithByteLength( + metadata: any, + metadataType: string, + maximumSize: number +): { packet: IOPacket; byteLength: number } | undefined { let metadataPacket: IOPacket | undefined = undefined; if (typeof metadata === "string") { @@ -33,5 +41,5 @@ export function handleMetadataPacket( throw new MetadataTooLargeError(`Metadata exceeds maximum size of ${maximumSize} bytes`); } - return metadataPacket; + return { packet: metadataPacket, byteLength }; } diff --git a/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts b/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts index 0905f7c768f..070be65f7ad 100644 --- a/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts +++ b/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts @@ -455,7 +455,10 @@ export class DeliverAlertService extends BaseService { error, }; - await this.#deliverWebhook(payload, webhookProperties.data); + await this.#deliverWebhook(payload, webhookProperties.data, { + webhookId: alert.channel.id, + runId: alert.taskRun.friendlyId, + }); break; } case "v2": { @@ -516,7 +519,10 @@ export class DeliverAlertService extends BaseService { }, }; - await this.#deliverWebhook(payload, webhookProperties.data); + await this.#deliverWebhook(payload, webhookProperties.data, { + webhookId: alert.channel.id, + runId: alert.taskRun.friendlyId, + }); break; } @@ -577,7 +583,9 @@ export class DeliverAlertService extends BaseService { vercel: this.#buildWebhookVercelObject(deploymentMeta.vercelDeploymentUrl), }; - await this.#deliverWebhook(payload, webhookProperties.data); + await this.#deliverWebhook(payload, webhookProperties.data, { + webhookId: alert.channel.id, + }); break; } case "v2": { @@ -616,7 +624,9 @@ export class DeliverAlertService extends BaseService { }, }; - await this.#deliverWebhook(payload, webhookProperties.data); + await this.#deliverWebhook(payload, webhookProperties.data, { + webhookId: alert.channel.id, + }); break; } @@ -671,7 +681,9 @@ export class DeliverAlertService extends BaseService { vercel: this.#buildWebhookVercelObject(deploymentMeta.vercelDeploymentUrl), }; - await this.#deliverWebhook(payload, webhookProperties.data); + await this.#deliverWebhook(payload, webhookProperties.data, { + webhookId: alert.channel.id, + }); break; } case "v2": { @@ -716,7 +728,9 @@ export class DeliverAlertService extends BaseService { }, }; - await this.#deliverWebhook(payload, webhookProperties.data); + await this.#deliverWebhook(payload, webhookProperties.data, { + webhookId: alert.channel.id, + }); break; } @@ -1017,7 +1031,11 @@ export class DeliverAlertService extends BaseService { } } - async #deliverWebhook<T>(payload: T, webhook: ProjectAlertWebhookProperties) { + async #deliverWebhook<T>( + payload: T, + webhook: ProjectAlertWebhookProperties, + context: { webhookId: string; runId?: string } + ) { const rawPayload = JSON.stringify(payload); const hashPayload = Buffer.from(rawPayload, "utf-8"); @@ -1046,15 +1064,17 @@ export class DeliverAlertService extends BaseService { }); if (!response.ok) { + // Never log the request/response body here: it is customer-controlled alert + // content and may include stack traces or other application data. logger.info("[DeliverAlert] Failed to send alert webhook", { status: response.status, statusText: response.statusText, - url: webhook.url, - body: payload, - signature, + urlHost: safeUrlHost(webhook.url), + webhookId: context.webhookId, + runId: context.runId, }); - throw new Error(`Failed to send alert webhook to ${webhook.url}`); + throw new Error(`Failed to send alert webhook to ${safeUrlHost(webhook.url)}`); } } @@ -1435,3 +1455,11 @@ function isWebAPIHTTPError(error: unknown): error is WebAPIHTTPError { function isWebAPIRateLimitedError(error: unknown): error is WebAPIRateLimitedError { return (error as WebAPIRateLimitedError).code === ErrorCode.RateLimitedError; } + +function safeUrlHost(url: string): string { + try { + return new URL(url).host; + } catch { + return "unknown"; + } +} diff --git a/packages/redis-worker/src/worker.ts b/packages/redis-worker/src/worker.ts index 58988b81896..b9240acaa63 100644 --- a/packages/redis-worker/src/worker.ts +++ b/packages/redis-worker/src/worker.ts @@ -140,7 +140,7 @@ class Worker<TCatalog extends WorkerCatalog> { > = new Map(); constructor(private options: WorkerOptions<TCatalog>) { - this.logger = options.logger ?? new Logger("Worker", "debug"); + this.logger = options.logger ?? new Logger("Worker", "debug", ["item"]); this.tracer = options.tracer ?? trace.getTracer(options.name); this.meter = options.meter ?? metrics.getMeter(options.name); @@ -608,7 +608,8 @@ class Worker<TCatalog extends WorkerCatalog> { this.logger.error("Unhandled error in processItem:", { error: err, workerId, - item, + id: queueItem.id, + job: queueItem.job, }); } ); @@ -933,11 +934,12 @@ class Worker<TCatalog extends WorkerCatalog> { const errorLogLevel = error && typeof error === "object" && "logLevel" in error ? error.logLevel : undefined; + // Never include the raw item/payload here: it is job data that may be + // customer-controlled. It is retrievable via `getJob(id)` if needed for triage. const logAttributes = { name: this.options.name, id, job, - item, visibilityTimeoutMs, error, errorMessage, @@ -994,7 +996,6 @@ class Worker<TCatalog extends WorkerCatalog> { name: this.options.name, id, job, - item, retryDate, retryDelay, visibilityTimeoutMs, @@ -1015,7 +1016,6 @@ class Worker<TCatalog extends WorkerCatalog> { name: this.options.name, id, job, - item, visibilityTimeoutMs, error: requeueError, } From 2f1734c858cc6500402f49f7fcac53179da13ce8 Mon Sep 17 00:00:00 2001 From: Chris Arderne <chris@trigger.dev> Date: Wed, 29 Jul 2026 17:59:47 +0100 Subject: [PATCH 37/67] fix(core,webapp): redact sensitive fields in logs by default and cap their size (#4401) --- apps/webapp/app/services/logger.server.ts | 24 +- .../webapp/test/logger.server.onError.test.ts | 69 ++++++ packages/core/src/logger.ts | 162 +++++++++++--- .../core/src/v3/utils/structuredLogger.ts | 6 +- packages/core/test/logger.test.ts | 211 ++++++++++++++++++ 5 files changed, 436 insertions(+), 36 deletions(-) create mode 100644 apps/webapp/test/logger.server.onError.test.ts create mode 100644 packages/core/test/logger.test.ts diff --git a/apps/webapp/app/services/logger.server.ts b/apps/webapp/app/services/logger.server.ts index b03802afefd..a31c7cc5a18 100644 --- a/apps/webapp/app/services/logger.server.ts +++ b/apps/webapp/app/services/logger.server.ts @@ -1,5 +1,5 @@ import type { LogLevel } from "@trigger.dev/core/logger"; -import { Logger } from "@trigger.dev/core/logger"; +import { Logger, redact } from "@trigger.dev/core/logger"; import { patchConsoleToTelnet, startTelnetLogServer } from "@trigger.dev/core/v3/telnetLogServer"; import { sensitiveDataReplacer } from "./sensitiveDataReplacer"; import { AsyncLocalStorage } from "async_hooks"; @@ -12,24 +12,40 @@ export function trace<T>(fields: Record<string, unknown>, fn: () => T): T { return currentFieldsStore.run(fields, fn); } +// The keys below aren't already in the Logger's default deny-list. Passing them here means the +// extra data sent to Sentry gets the same redaction as the stdout line, instead of bypassing it. +const SENTRY_EXTRA_FILTERED_KEYS = ["examples", "connectionString"]; + Logger.onError = (message, ...args) => { const error = extractErrorFromArgs(args); + const extra = redact(flattenArgs(args), SENTRY_EXTRA_FILTERED_KEYS) as Record<string, unknown>; if (error) { - captureException(error, { + captureException(redactError(error), { extra: { message, - ...flattenArgs(args), + ...extra, }, }); } else { captureMessage(message, { level: "error", - extra: flattenArgs(args), + extra, }); } }; +function redactError(error: Error): Error { + const redactedError = new Error(redact(error.message) as string); + redactedError.name = error.name; + + if (error.stack) { + redactedError.stack = redact(error.stack) as string; + } + + return redactedError; +} + function extractErrorFromArgs(args: Array<Record<string, unknown> | undefined>) { for (const arg of args) { if (arg && "error" in arg && arg.error instanceof Error) { diff --git a/apps/webapp/test/logger.server.onError.test.ts b/apps/webapp/test/logger.server.onError.test.ts new file mode 100644 index 00000000000..4aebabe6ac9 --- /dev/null +++ b/apps/webapp/test/logger.server.onError.test.ts @@ -0,0 +1,69 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const captureExceptionMock = vi.fn(); +const captureMessageMock = vi.fn(); + +vi.mock("@sentry/remix", () => ({ + captureException: captureExceptionMock, + captureMessage: captureMessageMock, +})); + +describe("logger.server Logger.onError", () => { + beforeEach(() => { + captureExceptionMock.mockClear(); + captureMessageMock.mockClear(); + vi.spyOn(console, "error").mockImplementation(() => {}); + }); + + it("redacts the extra payload sent to Sentry on the captureMessage path", async () => { + const { logger } = await import("~/services/logger.server"); + + logger.error("something failed", { + payload: { secret: "do-not-leak" }, + apiKey: "tr_prod_should_not_leak", + }); + + expect(captureMessageMock).toHaveBeenCalledTimes(1); + const [, options] = captureMessageMock.mock.calls[0] as [string, { extra: unknown }]; + + expect(JSON.stringify(options.extra)).not.toContain("do-not-leak"); + expect(JSON.stringify(options.extra)).not.toContain("tr_prod_should_not_leak"); + }); + + it("redacts the exception and extra payload sent to Sentry", async () => { + const { logger } = await import("~/services/logger.server"); + const error = new Error("tr_prod_should_not_leak"); + error.stack = "Bearer secret-token"; + + logger.error("boom", { + error, + payload: { secret: "do-not-leak" }, + }); + + expect(captureExceptionMock).toHaveBeenCalledTimes(1); + const [capturedError, options] = captureExceptionMock.mock.calls[0] as [ + Error, + { extra: unknown }, + ]; + + expect(capturedError).not.toBe(error); + expect(capturedError.message).not.toContain("tr_prod_should_not_leak"); + expect(capturedError.stack).not.toContain("secret-token"); + expect(JSON.stringify(options.extra)).not.toContain("do-not-leak"); + }); + + it("still forwards non-sensitive extra fields", async () => { + const { logger } = await import("~/services/logger.server"); + + logger.error("something failed", { runId: "run_123", keep: "this stays" }); + + expect(captureMessageMock).toHaveBeenCalledTimes(1); + const [, options] = captureMessageMock.mock.calls[0] as [ + string, + { extra: Record<string, unknown> }, + ]; + + expect(options.extra.runId).toBe("run_123"); + expect(options.extra.keep).toBe("this stays"); + }); +}); diff --git a/packages/core/src/logger.ts b/packages/core/src/logger.ts index 18ef4634a91..f3e51aef0ed 100644 --- a/packages/core/src/logger.ts +++ b/packages/core/src/logger.ts @@ -16,10 +16,53 @@ export type LogLevel = "log" | "error" | "warn" | "info" | "debug" | "verbose"; const logLevels: Array<LogLevel> = ["log", "error", "warn", "info", "debug", "verbose"]; +// Applied to every Logger instance, on top of whatever a caller passes in as `filteredKeys`. +// Keeps the previous "opt-in, per-instance" list from being the only thing standing between a +// logged object and a credential or piece of customer content that happens to share its name. +const DEFAULT_FILTERED_KEYS = [ + "authorization", + "token", + "apikey", + "secretkey", + "accesstoken", + "refreshtoken", + "password", + "jwt", + "payload", + "output", + "metadata", + "seedmetadata", + "input", + "email", + "headers", + "completedwaitpoints", +]; + +// Belt-and-braces value-shape check: catches secrets anywhere in values that land under a field +// name we didn't think to deny-list (a trigger.dev API key, bearer token, or OpenAI-style key). +const SECRET_VALUE_PATTERN = /(tr_[a-zA-Z0-9_-]{4,}|sk-[a-zA-Z0-9_-]{4,}|Bearer\s+\S+)/; + +// Per-field and per-structure caps so a single unbounded object (a run payload, a batch of +// items, a DB row) can't blow up log line size or CPU. Truncation keeps the field present and +// queryable rather than dropping it. +const MAX_STRING_LENGTH = 8192; +const MAX_ARRAY_LENGTH = 100; +const MAX_DEPTH = 10; + +function buildFilteredKeySet(filteredKeys: string[]): Set<string> { + const set = new Set(DEFAULT_FILTERED_KEYS); + + for (const key of filteredKeys) { + set.add(key.toLowerCase()); + } + + return set; +} + export class Logger { #name: string; readonly #level: number; - #filteredKeys: string[] = []; + #filteredKeys: Set<string> = new Set(DEFAULT_FILTERED_KEYS); #jsonReplacer?: (key: string, value: unknown) => unknown; #additionalFields: () => Record<string, unknown>; @@ -39,7 +82,7 @@ export class Logger { ) { this.#name = name; this.#level = logLevels.indexOf((env.TRIGGER_LOG_LEVEL ?? level) as LogLevel); - this.#filteredKeys = filteredKeys; + this.#filteredKeys = buildFilteredKeySet(filteredKeys); this.#jsonReplacer = createReplacer(jsonReplacer); this.#additionalFields = additionalFields ?? (() => ({})); } @@ -48,7 +91,7 @@ export class Logger { return new Logger( this.#name, logLevels[this.#level], - this.#filteredKeys, + Array.from(this.#filteredKeys), this.#jsonReplacer, () => ({ ...this.#additionalFields(), ...fields }) ); @@ -115,8 +158,8 @@ export class Logger { // Get the current context from trace if it exists const currentSpan = trace.getSpan(context.active()); - const structuredError = extractStructuredErrorFromArgs(...args); - const structuredMessage = extractStructuredMessageFromArgs(...args); + const structuredError = extractStructuredErrorFromArgs(this.#filteredKeys, ...args); + const structuredMessage = extractStructuredMessageFromArgs(this.#filteredKeys, ...args); const structuredLog = { ...structureArgs(safeJsonClone(args) as Record<string, unknown>[], this.#filteredKeys), @@ -153,38 +196,52 @@ export class Logger { // Detect if args is an error object // Or if args contains an error object at the "error" key // In both cases, return the error object as a structured error -function extractStructuredErrorFromArgs(...args: Array<Record<string, unknown> | undefined>) { - const error = args.find((arg) => arg instanceof Error) as Error | undefined; +// Run every field through the same filter/truncation used for the rest of the log line, so an +// error's message/stack/metadata (which can embed request or row data verbatim) gets the same +// treatment as everything else, instead of bypassing it. +function extractStructuredErrorFromArgs( + filteredKeys: Set<string>, + ...args: Array<Record<string, unknown> | undefined> +) { + const error = args.find((arg) => arg instanceof Error) as + | (Error & { metadata?: unknown }) + | undefined; if (error) { return { - message: error.message, - stack: error.stack, + message: filterKeys(error.message, filteredKeys), + stack: filterKeys(error.stack, filteredKeys), name: error.name, - metadata: "metadata" in error ? error.metadata : undefined, + metadata: "metadata" in error ? filterKeys(error.metadata, filteredKeys) : undefined, }; } const structuredError = args.find((arg) => arg?.error); if (structuredError && structuredError.error instanceof Error) { + const nestedError = structuredError.error as Error & { metadata?: unknown }; + return { - message: structuredError.error.message, - stack: structuredError.error.stack, - name: structuredError.error.name, - metadata: "metadata" in structuredError.error ? structuredError.error.metadata : undefined, + message: filterKeys(nestedError.message, filteredKeys), + stack: filterKeys(nestedError.stack, filteredKeys), + name: nestedError.name, + metadata: + "metadata" in nestedError ? filterKeys(nestedError.metadata, filteredKeys) : undefined, }; } return; } -function extractStructuredMessageFromArgs(...args: Array<Record<string, unknown> | undefined>) { +function extractStructuredMessageFromArgs( + filteredKeys: Set<string>, + ...args: Array<Record<string, unknown> | undefined> +) { // Check to see if there is a `message` key in the args, and if so, return it const structuredMessage = args.find((arg) => arg?.message); if (structuredMessage) { - return structuredMessage.message; + return filterKeys(structuredMessage.message, filteredKeys); } return; @@ -221,37 +278,65 @@ function safeJsonClone(obj: unknown) { } } -// If args is has a single item that is an object, return that object -function structureArgs(args: Array<Record<string, unknown>>, filteredKeys: string[] = []) { - if (!args) { +// `args` has already been through safeJsonClone, so this only has to filter/truncate it, not +// clone it again. If there's exactly one arg, return it directly (unwrapped) so it can be spread +// onto the structured log; otherwise filter every arg and return the array. Filtering runs +// regardless of arg count, so a multi-arg call gets the same redaction as the common single-arg +// case. +function structureArgs( + args: Array<Record<string, unknown>> | undefined, + filteredKeys: Set<string> = new Set() +) { + if (!args || args.length === 0) { return; } - if (args.length === 0) { - return; - } + const filteredArgs = args.map((arg) => filterKeys(arg, filteredKeys)); - if (args.length === 1 && typeof args[0] === "object") { - return filterKeys(JSON.parse(JSON.stringify(args[0], bigIntReplacer)), filteredKeys); + if (filteredArgs.length === 1) { + return filteredArgs[0]; } - return args; + return filteredArgs; } -// Recursively filter out keys from an object, including nested objects, and arrays -function filterKeys(obj: unknown, keys: string[]): any { +// Recursively filter out keys from an object, including nested objects and arrays. Also caps +// string length, array length and recursion depth, and redacts string values that look like a +// secret regardless of which key they were found under. +function filterKeys(obj: unknown, keys: Set<string>, depth = 0): any { + if (typeof obj === "string") { + if (SECRET_VALUE_PATTERN.test(obj)) { + return `[filtered ${prettyPrintBytes(obj)}]`; + } + + return truncateString(obj); + } + if (typeof obj !== "object" || obj === null) { return obj; } + if (depth >= MAX_DEPTH) { + return "[max depth exceeded]"; + } + if (Array.isArray(obj)) { - return obj.map((item) => filterKeys(item, keys)); + const isTruncated = obj.length > MAX_ARRAY_LENGTH; + const items = (isTruncated ? obj.slice(0, MAX_ARRAY_LENGTH) : obj).map((item) => + filterKeys(item, keys, depth + 1) + ); + + if (isTruncated) { + items.push(`[truncated ${obj.length - MAX_ARRAY_LENGTH} more items]`); + } + + return items; } const filteredObj: any = {}; for (const [key, value] of Object.entries(obj)) { - if (keys.includes(key)) { + if (keys.has(key.toLowerCase())) { if (value) { filteredObj[key] = `[filtered ${prettyPrintBytes(value)}]`; } else { @@ -260,12 +345,29 @@ function filterKeys(obj: unknown, keys: string[]): any { continue; } - filteredObj[key] = filterKeys(value, keys); + filteredObj[key] = filterKeys(value, keys, depth + 1); } return filteredObj; } +function truncateString(value: string): string { + if (value.length <= MAX_STRING_LENGTH) { + return value; + } + + return `${value.slice(0, MAX_STRING_LENGTH)}...[truncated ${ + value.length - MAX_STRING_LENGTH + } chars]`; +} + +// Runs a value through the same default-deny-list + truncation pipeline every Logger applies to +// its own log lines. For destinations that receive log arguments through a side channel (e.g. an +// error reporting `onError` hook) rather than through `Logger#structuredLog` itself. +export function redact(value: unknown, filteredKeys: string[] = []): unknown { + return filterKeys(value, buildFilteredKeySet(filteredKeys)); +} + function prettyPrintBytes(value: unknown): string { if (env.NODE_ENV === "production") { return "skipped size"; diff --git a/packages/core/src/v3/utils/structuredLogger.ts b/packages/core/src/v3/utils/structuredLogger.ts index c85d92a8609..e72cbfef475 100644 --- a/packages/core/src/v3/utils/structuredLogger.ts +++ b/packages/core/src/v3/utils/structuredLogger.ts @@ -1,3 +1,5 @@ +import { redact } from "../../logger.js"; + type StructuredArgs = (Record<string, unknown> | undefined)[]; export interface StructuredLogger { @@ -88,14 +90,14 @@ export class SimpleStructuredLogger implements StructuredLogger { level: string, ...args: StructuredArgs ) { - const structuredLog = { + const structuredLog = redact({ timestamp: new Date(), message, $name: this.name, $level: level, ...this.fields, ...(args.length === 1 ? args[0] : args), - }; + }) as Record<string, unknown>; if (SimpleStructuredLogger.onLog) { try { diff --git a/packages/core/test/logger.test.ts b/packages/core/test/logger.test.ts new file mode 100644 index 00000000000..94930acea2c --- /dev/null +++ b/packages/core/test/logger.test.ts @@ -0,0 +1,211 @@ +import { Logger, redact } from "../src/logger.js"; +import { SimpleStructuredLogger } from "../src/v3/utils/structuredLogger.js"; + +function captureLogLine(fn: () => void): Record<string, any> { + const spy = vi.spyOn(console, "info").mockImplementation(() => {}); + + try { + fn(); + expect(spy).toHaveBeenCalledTimes(1); + const [line] = spy.mock.calls[0] as [string]; + return JSON.parse(line); + } finally { + spy.mockRestore(); + } +} + +function captureErrorLogLine(fn: () => void): Record<string, any> { + const spy = vi.spyOn(console, "error").mockImplementation(() => {}); + + try { + fn(); + expect(spy).toHaveBeenCalledTimes(1); + const [line] = spy.mock.calls[0] as [string]; + return JSON.parse(line); + } finally { + spy.mockRestore(); + } +} + +describe("Logger redaction", () => { + it("redacts default deny-listed keys without any caller-supplied filteredKeys", () => { + const logger = new Logger("test", "info"); + + const line = captureLogLine(() => + logger.info("run started", { + payload: { secret: "value" }, + apiKey: "tr_prod_should_not_appear", + harmless: "keep me", + }) + ); + + expect(line.payload).toMatch(/^\[filtered/); + expect(line.apiKey).toMatch(/^\[filtered/); + expect(line.harmless).toBe("keep me"); + expect(JSON.stringify(line)).not.toContain("tr_prod_should_not_appear"); + }); + + it("matches deny-listed keys case-insensitively", () => { + const logger = new Logger("test", "info"); + + const line = captureLogLine(() => + logger.info("case check", { + ApiKey: "tr_prod_secret", + AUTHORIZATION: "Bearer abc123", + }) + ); + + expect(line.ApiKey).toMatch(/^\[filtered/); + expect(line.AUTHORIZATION).toMatch(/^\[filtered/); + }); + + it("recurses into nested objects and arrays", () => { + const logger = new Logger("test", "info"); + + const line = captureLogLine(() => + logger.info("nested check", { + run: { + items: [{ metadata: { token: "abc" } }, { metadata: { token: "def" } }], + }, + }) + ); + + for (const item of line.run.items) { + expect(item.metadata).toMatch(/^\[filtered/); + } + }); + + it("still honors caller-supplied filteredKeys in addition to the defaults", () => { + const logger = new Logger("test", "info", ["connectionString"]); + + const line = captureLogLine(() => + logger.info("db check", { connectionString: "postgres://user:pass@host/db" }) + ); + + expect(line.connectionString).toMatch(/^\[filtered/); + }); + + it("filters every argument, not only a single object argument", () => { + const logger = new Logger("test", "info"); + + const line = captureLogLine(() => + logger.info("two args", { first: "ok" }, { apiKey: "tr_prod_secret_value" }) + ); + + expect(JSON.stringify(line)).not.toContain("tr_prod_secret_value"); + }); + + it("redacts values that contain a secret even under a non-denied key", () => { + const logger = new Logger("test", "info"); + + const line = captureLogLine(() => + logger.info("value pattern check", { + someRandomField: "request failed with tr_live_abcdef123456", + anotherField: "authorization used Bearer some.jwt.value", + normalField: "just some text", + }) + ); + + expect(line.someRandomField).toMatch(/^\[filtered/); + expect(line.anotherField).toMatch(/^\[filtered/); + expect(line.normalField).toBe("just some text"); + }); + + it("redacts a structured message before assigning it to $message", () => { + const logger = new Logger("test", "info"); + + const line = captureLogLine(() => + logger.info("structured message check", { + message: "request failed with Bearer secret.jwt.value", + }) + ); + + expect(line.$message).toMatch(/^\[filtered/); + expect(JSON.stringify(line)).not.toContain("secret.jwt.value"); + }); + + it("truncates strings longer than the per-field cap", () => { + const logger = new Logger("test", "info"); + const longValue = "a".repeat(20_000); + + const line = captureLogLine(() => logger.info("truncation check", { longValue })); + + expect(line.longValue.length).toBeLessThan(longValue.length); + expect(line.longValue).toContain("[truncated"); + }); + + it("truncates arrays longer than the max array length", () => { + const logger = new Logger("test", "info"); + const bigArray = Array.from({ length: 500 }, (_, i) => i); + + const line = captureLogLine(() => logger.info("array truncation check", { bigArray })); + + expect(line.bigArray.length).toBeLessThan(bigArray.length); + expect(line.bigArray[line.bigArray.length - 1]).toContain("truncated"); + }); + + it("runs error metadata through the same key-based redaction as everything else", () => { + const logger = new Logger("test", "info"); + + const error = new Error("a plain failure message") as Error & { metadata?: unknown }; + error.metadata = { apiKey: "tr_prod_should_not_leak" }; + + const line = captureErrorLogLine(() => logger.error("boom", { error })); + + expect(line.error.message).toBe("a plain failure message"); + expect(line.error.metadata.apiKey).toMatch(/^\[filtered/); + expect(JSON.stringify(line)).not.toContain("tr_prod_should_not_leak"); + }); + + it("redacts an error message whose entire value is a bare secret", () => { + const logger = new Logger("test", "info"); + + const error = new Error("tr_prod_secret_value_leaked"); + + const line = captureErrorLogLine(() => logger.error("boom", { error })); + + expect(line.error.message).toMatch(/^\[filtered/); + }); +}); + +describe("SimpleStructuredLogger redaction", () => { + it("redacts fields and arguments with the default deny-list", () => { + const logger = new SimpleStructuredLogger("test"); + + const line = captureLogLine(() => + logger.child({ headers: { authorization: "Bearer should-not-appear" } }).info("run started", { + payload: { secret: "value" }, + apiKey: "tr_prod_should_not_appear", + harmless: "keep me", + }) + ); + + expect(line.headers).toMatch(/^\[filtered/); + expect(line.payload).toMatch(/^\[filtered/); + expect(line.apiKey).toMatch(/^\[filtered/); + expect(line.harmless).toBe("keep me"); + expect(JSON.stringify(line)).not.toContain("should-not-appear"); + }); +}); + +describe("redact()", () => { + it("applies the same default deny-list and truncation as the Logger", () => { + const result = redact({ + apiKey: "tr_prod_secret_value", + email: "user@example.com", + keep: "this stays", + }) as Record<string, unknown>; + + expect(result.apiKey).toMatch(/^\[filtered/); + expect(result.email).toMatch(/^\[filtered/); + expect(result.keep).toBe("this stays"); + }); + + it("merges in caller-supplied filteredKeys", () => { + const result = redact({ connectionString: "postgres://user:pass@host/db" }, [ + "connectionString", + ]) as Record<string, unknown>; + + expect(result.connectionString).toMatch(/^\[filtered/); + }); +}); From 6e5f0f0fe74fceb2f8462e7fd354790572d5f03c Mon Sep 17 00:00:00 2001 From: Eric Allam <eric@trigger.dev> Date: Thu, 30 Jul 2026 09:04:15 +0100 Subject: [PATCH 38/67] fix(webapp,clickhouse): stop invalid customer queries alerting, and isolate Sentry scope per request (#4372) ## Summary A query sent to the query API with a typo in it, like a column name that does not exist, was being reported as a server error. That put customer SQL mistakes into our error alerting, where they made up almost all of the volume on one of our noisiest alerts, and it drowned out the failures that are actually ours to fix. This makes the level match who is at fault, and fixes two related problems found alongside it. ## Invalid queries are the caller's, not ours The query API route already got this right. It checks for `QueryError`, logs at warn, and returns a 400, with a comment saying the system handles it gracefully and no alert is needed. The layer underneath ignored that. `executeTSQL` logged every exception out of its catch block at error, including the compile failures the route was about to turn into a 400, and error-level logs are forwarded to error reporting. The TSQL package already draws the line we need: ```ts export class ExposedTSQLError extends BaseTSQLError { /** An exception that can be exposed to the user. */ } export class InternalTSQLError extends BaseTSQLError { /** An internal exception in the TSQL engine. */ } ``` `SyntaxError` and `QueryError` extend the first. So the catch block now branches on `ExposedTSQLError` and logs those at warn, keeping error for `InternalTSQLError` and anything unanticipated, which is a genuine compiler bug. ## SQL the caller wrote is their mistake, not ours The same asymmetry showed up one level down. A query that compiles fine can still be rejected by ClickHouse at execution, and most of those rejections mean the caller's SQL is wrong rather than that we generated something bad. This is where the volume actually is. Checking production, one error group alone, a missing `GROUP BY` on the public query API (`NOT_AN_AGGREGATE`), accounts for over a million events across hundreds of users. It is by far the largest error group in the project, and classifying only by resource limit would have left every one of those at error level. So rejections are split three ways in `ClickhouseClient`, which is the only place holding the parsed `ClickHouseError` and its symbolic type. By the time the error reaches `executeTSQL` it has been wrapped and the type is gone, and the type never appears in the message text, so it cannot be recovered by string matching. - **Resource limits** (memory ceiling, timeout, row/byte caps) log at warn. The query is valid, it just asked for more than it is allowed to spend. - **Invalid SQL** (`NOT_AN_AGGREGATE`, `UNKNOWN_IDENTIFIER`, `SYNTAX_ERROR`, the type and parse families) logs at warn **only when the caller wrote the SQL**. - **Everything else** keeps alerting. That gate matters. The client is shared, so the identical rejection on TRQL *we* generated is our bug and has to stay at error. Callers opt in with `userAuthoredQuery`: | caller | who wrote the SQL | opts in | | --- | --- | --- | | public query API | the customer | yes | | query editor | the customer | yes | | agent charts | the agent's model | yes | | built-in dashboard tiles | us, in code | no | | queue metric cards | us, in code | no | | health report | us, in code | no | The agent is the one judgement call. Its TRQL is not typed by a person, but it is also not something a code fix makes correct, so a query it gets wrong is not worth waking anyone for. The same endpoint serves built-in tiles whose TRQL we do write, so the opt-in lives with the caller rather than the route. Separately, when one of these queries did fail, the log recorded the generated ClickHouse SQL but not the query the caller actually wrote, which made the reports hard to act on. `queryWithStats` takes an optional `logFields` that `executeTSQL` uses to attach the original TSQL. ## Events were attributed to the wrong request Chasing the above turned up something broader: only a tenth of the events on that alert pointed at the query API. The rest were pinned to unrelated requests that happened to be in flight at the same time, so the alert looked like the trigger endpoint was failing. `Sentry.init` runs with `skipOpenTelemetrySetup: true`, because we register our own OTel pipeline. That skips `initOpenTelemetry`, and one of the things it does is: ```js api.context.setGlobalContextManager(new SentryContextManager()); ``` The async-context strategy is still installed, but `withIsolationScope` only marks the OTel context and delegates the actual fork to that context manager: ```js // "We depend on the otelContextManager to handle the context/hub" return api.context.with(ctx.setValue(SENTRY_FORK_ISOLATION_SCOPE_CONTEXT_KEY, true), ...) ``` `provider.register()` installed a plain `AsyncLocalStorageContextManager`, which does not know that key. The lookup found no scopes on the context and fell back to the process-global default isolation scope, so every request wrote its request data into the same object and the last writer won. The tracer now registers `SentryContextManager`, which subclasses `AsyncLocalStorageContextManager`, so OTel behaviour is unchanged. It is also registered on the path where tracing is disabled, which previously never called `register()` at all and so had no context manager of its own. Tenant tags were always correct, because those come from our own async local storage rather than the isolation scope. That is why the attribution being wrong was not obvious. This affects every error report the webapp sends, not just the query API. ## Verification `internal-packages/clickhouse`: 76 tests pass, including eight covering each level decision against a real ClickHouse container. Three pairs pin the gate open and shut at both layers: an invalid query, a compile failure, and a real limit breach driven with `max_rows_to_read` each log at warn with `userAuthoredQuery` and at error without it. The isolation fix has a test that reproduces the leak before asserting the fix. Two overlapping requests each tag their own isolation scope; with the plain context manager the slower one reads back the other's tag, and with `SentryContextManager` each reads back its own. Measured separately against a faithful reproduction of the server's wiring (own OTel pipeline, CommonJS entry) at 200 concurrent requests: per-request attribution goes from 0.5% to 100%, while span nesting, context propagation across awaits, and distinct trace IDs are identical before and after. --- .server-changes/sentry-request-attribution.md | 6 + .../tsql-query-error-log-levels.md | 6 + .../components/dashboard-agent/AgentChart.tsx | 1 + .../route.tsx | 1 + apps/webapp/app/routes/api.v1.query.ts | 1 + apps/webapp/app/routes/resources.metric.tsx | 3 + .../app/services/queryService.server.ts | 7 + apps/webapp/app/v3/tracer.server.ts | 26 ++- .../test/sentryRequestIsolation.test.ts | 51 ++++ .../clickhouse/src/client/client.ts | 103 +++++++- .../clickhouse/src/client/tsql.ts | 23 +- .../clickhouse/src/client/types.ts | 10 + internal-packages/clickhouse/src/tsql.test.ts | 220 ++++++++++++++++++ 13 files changed, 446 insertions(+), 12 deletions(-) create mode 100644 .server-changes/sentry-request-attribution.md create mode 100644 .server-changes/tsql-query-error-log-levels.md create mode 100644 apps/webapp/test/sentryRequestIsolation.test.ts diff --git a/.server-changes/sentry-request-attribution.md b/.server-changes/sentry-request-attribution.md new file mode 100644 index 00000000000..d42abf78d09 --- /dev/null +++ b/.server-changes/sentry-request-attribution.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Fixed error reports being attributed to the wrong request when several requests were in flight at once. diff --git a/.server-changes/tsql-query-error-log-levels.md b/.server-changes/tsql-query-error-log-levels.md new file mode 100644 index 00000000000..1aaedba3b76 --- /dev/null +++ b/.server-changes/tsql-query-error-log-levels.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Invalid queries sent to the query API are no longer treated as internal errors, and a query that does fail is now recorded together with the query text that produced it. diff --git a/apps/webapp/app/components/dashboard-agent/AgentChart.tsx b/apps/webapp/app/components/dashboard-agent/AgentChart.tsx index 6e082231eb6..dbd1297c58f 100644 --- a/apps/webapp/app/components/dashboard-agent/AgentChart.tsx +++ b/apps/webapp/app/components/dashboard-agent/AgentChart.tsx @@ -67,6 +67,7 @@ export function AgentChart({ block }: { block: ChartBlock }) { period: block.period ?? null, from: null, to: null, + userAuthoredQuery: true, }), signal: controller.signal, }) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx index 848546a518f..c16037f5e10 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx @@ -183,6 +183,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const queryResult = await executeQuery({ name: "query-page", query, + userAuthoredQuery: true, scope, organizationId: project.organizationId, projectId: project.id, diff --git a/apps/webapp/app/routes/api.v1.query.ts b/apps/webapp/app/routes/api.v1.query.ts index b3459fa51c8..2f4011455ba 100644 --- a/apps/webapp/app/routes/api.v1.query.ts +++ b/apps/webapp/app/routes/api.v1.query.ts @@ -54,6 +54,7 @@ const { action, loader } = createActionApiRoute( const queryResult = await executeQuery({ name: "api-query", query, + userAuthoredQuery: true, scope: scope as QueryScope, organizationId: env.organization.id, projectId: env.project.id, diff --git a/apps/webapp/app/routes/resources.metric.tsx b/apps/webapp/app/routes/resources.metric.tsx index 5bf0ed693ad..1808f6107e4 100644 --- a/apps/webapp/app/routes/resources.metric.tsx +++ b/apps/webapp/app/routes/resources.metric.tsx @@ -52,6 +52,7 @@ const MetricWidgetQuery = z.object({ tags: z.array(z.string()).optional(), // Opt into server-side gap fill (carry-forward for gauges, zero-fill for counters). fillGaps: z.boolean().optional(), + userAuthoredQuery: z.boolean().optional(), }); export const action = async ({ request }: ActionFunctionArgs) => { @@ -88,6 +89,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { providers, tags: _tags, fillGaps, + userAuthoredQuery, } = submission.data; // Check they should be able to access it @@ -126,6 +128,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { operations, providers, fillGaps, + userAuthoredQuery, // Set higher concurrency if many widgets are on screen at once customOrgConcurrencyLimit: env.METRIC_WIDGET_DEFAULT_ORG_CONCURRENCY_LIMIT, }); diff --git a/apps/webapp/app/services/queryService.server.ts b/apps/webapp/app/services/queryService.server.ts index 3a34f82cf1d..bcceca25bbb 100644 --- a/apps/webapp/app/services/queryService.server.ts +++ b/apps/webapp/app/services/queryService.server.ts @@ -100,6 +100,13 @@ export type ExecuteQueryOptions<TOut extends z.ZodSchema> = Omit< }; /** Custom per-org concurrency limit (overrides default) */ customOrgConcurrencyLimit?: number; + /** + * Set when the caller wrote `query` themselves, as on the public query API and + * the query editor. ClickHouse rejecting their SQL is then their mistake, so it + * is logged as a warning instead of raising an alert. Leave unset for TRQL we + * generate, where the same rejection is a bug worth alerting on. + */ + userAuthoredQuery?: boolean; }; /** diff --git a/apps/webapp/app/v3/tracer.server.ts b/apps/webapp/app/v3/tracer.server.ts index 3b924ff8a19..87e0877091f 100644 --- a/apps/webapp/app/v3/tracer.server.ts +++ b/apps/webapp/app/v3/tracer.server.ts @@ -1,6 +1,7 @@ import { type Attributes, type Context, + context as otelContext, createContextKey, DiagConsoleLogger, DiagLogLevel, @@ -14,6 +15,7 @@ import { metrics, type Meter, } from "@opentelemetry/api"; +import sentryRemix from "@sentry/remix"; import { logs, SeverityNumber } from "@opentelemetry/api-logs"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http"; @@ -209,10 +211,32 @@ function getResource() { return baseResource.merge(detectedResource); } +/** + * Sentry's `withIsolationScope` only marks the OTel context; the fork itself is + * done by Sentry's context manager. We pass `skipOpenTelemetrySetup: true` to + * `Sentry.init` because we run our own OTel pipeline, which also skips the + * `setGlobalContextManager(new SentryContextManager())` that Sentry would + * otherwise do. Registering it here is what keeps per-request scopes (and so + * the request attributed to each Sentry event) from leaking between concurrent + * requests. It extends `AsyncLocalStorageContextManager`, so OTel behaviour is + * unchanged. + * + * Reached through the default export because `@sentry/remix` is CommonJS and + * Node's ESM loader does not detect this transitively re-exported name, so a + * named import resolves at build time and then fails when the server boots. + */ +function createContextManager() { + return new sentryRemix.SentryContextManager(); +} + function setupTelemetry() { if (env.INTERNAL_OTEL_TRACE_DISABLED === "1") { console.log(`🔦 Tracer disabled, returning a noop tracer`); + const contextManager = createContextManager(); + contextManager.enable(); + otelContext.setGlobalContextManager(contextManager); + return { tracer: trace.getTracer("trigger.dev", "3.3.12"), logger: logs.getLogger("trigger.dev", "3.3.12"), @@ -300,7 +324,7 @@ function setupTelemetry() { ); } - provider.register(); + provider.register({ contextManager: createContextManager() }); let instrumentations: Instrumentation[] = [ new AwsSdkInstrumentation({ diff --git a/apps/webapp/test/sentryRequestIsolation.test.ts b/apps/webapp/test/sentryRequestIsolation.test.ts new file mode 100644 index 00000000000..ebcb0af6929 --- /dev/null +++ b/apps/webapp/test/sentryRequestIsolation.test.ts @@ -0,0 +1,51 @@ +import { context } from "@opentelemetry/api"; +import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node"; +import * as Sentry from "@sentry/remix"; +import sentryRemix from "@sentry/remix"; +import { afterEach, beforeAll, describe, expect, it } from "vitest"; + +/** + * Two overlapping requests, each tagging its own isolation scope, mirroring what + * `SentryHttpInstrumentation` does per incoming request. Returns what each one + * reads back after the other has started. + */ +async function raceTwoRequests(): Promise<Record<string, unknown>> { + const observed: Record<string, unknown> = {}; + + const handleRequest = (name: string, holdMs: number) => + Sentry.withIsolationScope(async () => { + Sentry.getIsolationScope().setTag("request", name); + await new Promise((resolve) => setTimeout(resolve, holdMs)); + observed[name] = Sentry.getIsolationScope().getScopeData().tags.request; + }); + + await Promise.all([handleRequest("slow", 30), handleRequest("fast", 5)]); + + return observed; +} + +describe("Sentry request isolation", () => { + beforeAll(() => { + Sentry.init({ dsn: undefined, defaultIntegrations: false, skipOpenTelemetrySetup: true }); + }); + + afterEach(() => { + context.disable(); + }); + + it("leaks the isolation scope between concurrent requests without SentryContextManager", async () => { + new NodeTracerProvider().register(); + + const observed = await raceTwoRequests(); + + expect(observed).toEqual({ slow: "fast", fast: "fast" }); + }); + + it("keeps each request's isolation scope separate with SentryContextManager", async () => { + new NodeTracerProvider().register({ contextManager: new sentryRemix.SentryContextManager() }); + + const observed = await raceTwoRequests(); + + expect(observed).toEqual({ slow: "slow", fast: "fast" }); + }); +}); diff --git a/internal-packages/clickhouse/src/client/client.ts b/internal-packages/clickhouse/src/client/client.ts index d91f86b428f..ef18883745b 100644 --- a/internal-packages/clickhouse/src/client/client.ts +++ b/internal-packages/clickhouse/src/client/client.ts @@ -171,13 +171,15 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { ); if (clickhouseError) { - this.logger.error("Error querying clickhouse", { + const errorLogFields = { name: req.name, error: clickhouseError, query: req.query, params, queryId, - }); + }; + + this.logger.error("Error querying clickhouse", errorLogFields); recordClickhouseError(span, clickhouseError); @@ -260,6 +262,16 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { * These will be merged with the default settings. */ settings?: ClickHouseSettings; + /** + * Extra fields to attach to the error log if the query fails. Use this to + * record what produced the SQL, e.g. the TSQL a caller actually wrote. + */ + logFields?: Record<string, unknown>; + /** + * Set when the SQL originates from whoever made the request rather than + * from us. Invalid-SQL rejections are then their mistake, not a bug. + */ + userAuthoredQuery?: boolean; }): ClickhouseQueryWithStatsFunction<z.input<TIn>, z.output<TOut>> { return async (params, options) => { const queryId = randomUUID(); @@ -320,13 +332,25 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { ); if (clickhouseError) { - this.logger.error("Error querying clickhouse", { + const errorLogFields = { + ...req.logFields, name: req.name, error: clickhouseError, query: req.query, params, queryId, - }); + }; + + switch (classifyClickhouseError(clickhouseError, req.userAuthoredQuery)) { + case "quota": + this.logger.warn("Query exceeded a ClickHouse limit", errorLogFields); + break; + case "invalid-sql": + this.logger.warn("ClickHouse rejected an invalid query", errorLogFields); + break; + default: + this.logger.error("Error querying clickhouse", errorLogFields); + } recordClickhouseError(span, clickhouseError); @@ -453,13 +477,15 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { ); if (clickhouseError) { - this.logger.error("Error querying clickhouse", { + const errorLogFields = { name: req.name, error: clickhouseError, query: req.query, params, queryId, - }); + }; + + this.logger.error("Error querying clickhouse", errorLogFields); recordClickhouseError(span, clickhouseError); @@ -599,13 +625,15 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { span.setAttributes({ "clickhouse.rows": rowCount }); } catch (error) { - self.logger.error("Error streaming clickhouse", { + const errorLogFields = { name: req.name, error, query: req.query, params, queryId, - }); + }; + + self.logger.error("Error streaming clickhouse", errorLogFields); if (error instanceof Error) { recordClickhouseError(span, error); @@ -1001,6 +1029,65 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { } } +/** + * ClickHouse error types raised by a query that is valid but asks for more than + * it is allowed to spend. Only downgraded for SQL the caller wrote: a runaway + * query we generated is our bug and still has to alert. + */ +const CLICKHOUSE_QUOTA_ERROR_TYPES = new Set([ + "MEMORY_LIMIT_EXCEEDED", + "TIMEOUT_EXCEEDED", + "TOO_SLOW", + "TOO_MANY_ROWS", + "TOO_MANY_BYTES", + "TOO_MANY_ROWS_OR_BYTES", +]); + +/** + * ClickHouse error types that mean the SQL itself is wrong. Only treated as the + * caller's fault when the query was written by the caller — the same error on a + * query we generated is our bug and has to keep alerting. + */ +const CLICKHOUSE_INVALID_SQL_ERROR_TYPES = new Set([ + "NOT_AN_AGGREGATE", + "ILLEGAL_AGGREGATION", + "UNKNOWN_IDENTIFIER", + "UNKNOWN_FUNCTION", + "UNKNOWN_TABLE", + "AMBIGUOUS_COLUMN_NAME", + "MULTIPLE_EXPRESSIONS_FOR_ALIAS", + "SYNTAX_ERROR", + "BAD_ARGUMENTS", + "TYPE_MISMATCH", + "NO_COMMON_TYPE", + "ILLEGAL_TYPE_OF_ARGUMENT", + "ILLEGAL_COLUMN", + "CANNOT_CONVERT_TYPE", + "CANNOT_PARSE_TEXT", + "CANNOT_PARSE_NUMBER", + "CANNOT_PARSE_DATE", + "CANNOT_PARSE_DATETIME", + "CANNOT_PARSE_INPUT_ASSERTION_FAILED", +]); + +type ClickhouseErrorCategory = "quota" | "invalid-sql" | "fault"; + +function classifyClickhouseError( + error: Error, + userAuthoredQuery: boolean | undefined +): ClickhouseErrorCategory { + if (!userAuthoredQuery || !(error instanceof ClickHouseError) || error.type === undefined) { + return "fault"; + } + if (CLICKHOUSE_QUOTA_ERROR_TYPES.has(error.type)) { + return "quota"; + } + if (CLICKHOUSE_INVALID_SQL_ERROR_TYPES.has(error.type)) { + return "invalid-sql"; + } + return "fault"; +} + function recordClickhouseError(span: Span, error: Error): void { if (error instanceof ClickHouseError) { span.setAttributes({ diff --git a/internal-packages/clickhouse/src/client/tsql.ts b/internal-packages/clickhouse/src/client/tsql.ts index ddf1d059b97..2cf586c36f1 100644 --- a/internal-packages/clickhouse/src/client/tsql.ts +++ b/internal-packages/clickhouse/src/client/tsql.ts @@ -8,6 +8,7 @@ import type { ClickHouseSettings } from "@clickhouse/client"; import { compileTSQL, + ExposedTSQLError, type OutputColumnMetadata, sanitizeErrorMessage, transformResults, @@ -113,6 +114,11 @@ export interface ExecuteTSQLOptions<TOut extends z.ZodSchema> { * (counters zero-fill, gauges carry forward). Off by default. */ fillGaps?: boolean; + /** + * Set when `query` was written by whoever made the request rather than by us. + * A rejection of their SQL is then their mistake, not a bug on our side. + */ + userAuthoredQuery?: boolean; } /** @@ -213,6 +219,8 @@ export async function executeTSQL<TOut extends z.ZodSchema>( // EXPLAIN returns rows with an 'explain' column schema: isExplain ? z.object({ explain: z.string() }) : options.schema, settings: options.clickhouseSettings, + logFields: { tsql: options.query }, + userAuthoredQuery: options.userAuthoredQuery, }); const [error, result] = await queryFn(params); @@ -246,6 +254,8 @@ export async function executeTSQL<TOut extends z.ZodSchema>( params: z.record(z.any()), schema: z.object({ explain: z.string() }), settings: options.clickhouseSettings, + logFields: { tsql: options.query }, + userAuthoredQuery: options.userAuthoredQuery, }); const [additionalError, additionalResult] = await additionalQueryFn(params); @@ -303,14 +313,21 @@ export async function executeTSQL<TOut extends z.ZodSchema>( } catch (error) { const errorMessage = error instanceof Error ? error.message : "Unknown error"; - // Log TSQL compilation or unexpected errors (with original message for debugging) - logger.error("[TSQL] Query error", { + const logFields = { name: options.name, error: errorMessage, tsql: options.query, generatedSql: generatedSql ?? "(compilation failed)", generatedParams: generatedParams ?? {}, - }); + }; + + const callerWroteABadQuery = options.userAuthoredQuery && error instanceof ExposedTSQLError; + + if (callerWroteABadQuery) { + logger.warn("[TSQL] Invalid query", logFields); + } else { + logger.error("[TSQL] Query error", logFields); + } // Sanitize error message to show TSQL names instead of ClickHouse internals const sanitizedMessage = sanitizeErrorMessage(errorMessage, options.tableSchema); diff --git a/internal-packages/clickhouse/src/client/types.ts b/internal-packages/clickhouse/src/client/types.ts index d785ab34ebb..4bfa6dc4662 100644 --- a/internal-packages/clickhouse/src/client/types.ts +++ b/internal-packages/clickhouse/src/client/types.ts @@ -135,6 +135,16 @@ export interface ClickhouseReader { * These will be merged with the default settings. */ settings?: ClickHouseSettings; + /** + * Extra fields to attach to the error log if the query fails. Use this to + * record what produced the SQL, e.g. the TSQL a caller actually wrote. + */ + logFields?: Record<string, unknown>; + /** + * Set when the SQL originates from whoever made the request rather than + * from us. Invalid-SQL rejections are then their mistake, not a bug. + */ + userAuthoredQuery?: boolean; }): ClickhouseQueryWithStatsFunction<z.input<TIn>, z.output<TOut>>; queryFast<TOut extends Record<string, any>, TParams extends Record<string, any>>(req: { diff --git a/internal-packages/clickhouse/src/tsql.test.ts b/internal-packages/clickhouse/src/tsql.test.ts index b0a93bfe704..ff16508fdc7 100644 --- a/internal-packages/clickhouse/src/tsql.test.ts +++ b/internal-packages/clickhouse/src/tsql.test.ts @@ -1,4 +1,5 @@ import { clickhouseTest } from "@internal/testcontainers"; +import type { MockInstance } from "vitest"; import { z } from "zod"; import { ClickhouseClient } from "./client/client.js"; import { executeTSQL, createTSQLExecutor, type TableSchema } from "./client/tsql.js"; @@ -1609,3 +1610,222 @@ describe("Field Mapping Tests", () => { expect(result?.rows?.map((r) => r.run_id).sort()).toEqual(["run_fm_in1", "run_fm_in2"]); }); }); + +describe("TSQL Error Log Levels", () => { + let warnSpy: MockInstance<typeof console.warn>; + let errorSpy: MockInstance<typeof console.error>; + + beforeEach(() => { + warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + function logged(spy: MockInstance<typeof console.warn>): string { + return spy.mock.calls.map(([line]) => String(line)).join("\n"); + } + + clickhouseTest("logs an unknown column as a warning", async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-unknown-column", + query: "SELECT nope FROM task_runs", + schema: z.object({ nope: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + userAuthoredQuery: true, + }); + + expect(error).not.toBeNull(); + expect(logged(warnSpy)).toContain("[TSQL] Invalid query"); + expect(logged(errorSpy)).not.toContain("[TSQL] Query error"); + }); + + clickhouseTest("logs a syntax error as a warning", async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-syntax-error", + query: "SELECT FROM WHERE", + schema: z.object({ run_id: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + userAuthoredQuery: true, + }); + + expect(error).not.toBeNull(); + expect(logged(warnSpy)).toContain("[TSQL] Invalid query"); + expect(logged(errorSpy)).not.toContain("[TSQL] Query error"); + }); + + clickhouseTest( + "logs a query ClickHouse rejects at execution as an error, with the TSQL that produced it", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-execution-error", + query: "SELECT toDateTime(tags) AS bad FROM task_runs", + schema: z.object({ bad: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + }); + + expect(error).not.toBeNull(); + expect(logged(errorSpy)).toContain("Error querying clickhouse"); + expect(logged(errorSpy)).toContain("SELECT toDateTime(tags) AS bad FROM task_runs"); + } + ); + + clickhouseTest( + "keeps a compile failure on TRQL we generated at error level", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-internal-compile-error", + query: "SELECT nope FROM task_runs", + schema: z.object({ nope: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + }); + + expect(error).not.toBeNull(); + expect(logged(errorSpy)).toContain("[TSQL] Query error"); + expect(logged(warnSpy)).not.toContain("[TSQL] Invalid query"); + } + ); + + clickhouseTest( + "logs invalid caller-written SQL as a warning", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-user-authored-invalid", + query: "SELECT status, sum(is_test) AS n FROM task_runs", + schema: z.object({ status: z.string(), n: z.number() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + userAuthoredQuery: true, + }); + + expect(error).not.toBeNull(); + expect(logged(warnSpy)).toContain("ClickHouse rejected an invalid query"); + expect(logged(errorSpy)).not.toContain("Error querying clickhouse"); + } + ); + + clickhouseTest( + "keeps invalid SQL we generated at error level", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const [error] = await executeTSQL(client, { + name: "test-internal-invalid", + query: "SELECT status, sum(is_test) AS n FROM task_runs", + schema: z.object({ status: z.string(), n: z.number() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + }); + + expect(error).not.toBeNull(); + expect(logged(errorSpy)).toContain("Error querying clickhouse"); + expect(logged(warnSpy)).not.toContain("ClickHouse rejected an invalid query"); + } + ); + + clickhouseTest("logs a ClickHouse limit breach as a warning", async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + await insertTaskRuns(client, { async_insert: 0 })([ + createTaskRun({ run_id: "run_limit1" }), + createTaskRun({ run_id: "run_limit2" }), + createTaskRun({ run_id: "run_limit3" }), + ]); + + const [error] = await executeTSQL(client, { + name: "test-resource-limit", + query: "SELECT run_id FROM task_runs", + schema: z.object({ run_id: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + userAuthoredQuery: true, + clickhouseSettings: { max_rows_to_read: "1" }, + }); + + expect(error).not.toBeNull(); + expect(logged(warnSpy)).toContain("Query exceeded a ClickHouse limit"); + expect(logged(errorSpy)).not.toContain("Error querying clickhouse"); + }); + + clickhouseTest( + "keeps a limit breach on a query we generated at error level", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + await insertTaskRuns(client, { async_insert: 0 })([ + createTaskRun({ run_id: "run_intlimit1" }), + createTaskRun({ run_id: "run_intlimit2" }), + createTaskRun({ run_id: "run_intlimit3" }), + ]); + + const [error] = await executeTSQL(client, { + name: "test-internal-resource-limit", + query: "SELECT run_id FROM task_runs", + schema: z.object({ run_id: z.string() }), + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, + tableSchema: [taskRunsSchema], + clickhouseSettings: { max_rows_to_read: "1" }, + }); + + expect(error).not.toBeNull(); + expect(logged(errorSpy)).toContain("Error querying clickhouse"); + expect(logged(warnSpy)).not.toContain("Query exceeded a ClickHouse limit"); + } + ); +}); From 86b948b47a14bb18f3fd7451926fe2a985ed29f9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 10:12:14 +0100 Subject: [PATCH 39/67] chore: release v4.5.9 (#4408) --- .../mcp-server-starts-without-wizard.md | 5 ---- .changeset/public-token-expiration-seconds.md | 5 ---- .changeset/report-health.md | 12 ---------- .changeset/silent-build-logs.md | 5 ---- .changeset/target-cli-notifications.md | 5 ---- .../agent-detail-metrics-layout.md | 6 ----- ...invite-role-change-for-existing-members.md | 6 ----- .server-changes/internal-api-origin-flag.md | 6 ----- .../paginate-concurrency-keys-table.md | 6 ----- .../queue-concurrency-percent-limits.md | 6 ----- .server-changes/queue-metrics-dashboard.md | 6 ----- .../reject-queue-limit-above-env-limit.md | 6 ----- .../remove-unused-trace-sync-route.md | 6 ----- .server-changes/sentry-request-attribution.md | 6 ----- .../tsql-query-error-log-levels.md | 6 ----- hosting/k8s/helm/Chart.yaml | 4 ++-- packages/build/CHANGELOG.md | 7 ++++++ packages/build/package.json | 4 ++-- packages/cli-v3/CHANGELOG.md | 20 ++++++++++++++++ packages/cli-v3/package.json | 8 +++---- packages/core/CHANGELOG.md | 12 ++++++++++ packages/core/package.json | 2 +- packages/python/CHANGELOG.md | 9 +++++++ packages/python/package.json | 12 +++++----- packages/react-hooks/CHANGELOG.md | 7 ++++++ packages/react-hooks/package.json | 4 ++-- packages/redis-worker/CHANGELOG.md | 7 ++++++ packages/redis-worker/package.json | 4 ++-- packages/rsc/CHANGELOG.md | 7 ++++++ packages/rsc/package.json | 6 ++--- packages/schema-to-json/CHANGELOG.md | 7 ++++++ packages/schema-to-json/package.json | 2 +- packages/trigger-sdk/CHANGELOG.md | 8 +++++++ packages/trigger-sdk/package.json | 4 ++-- pnpm-lock.yaml | 24 +++++++++---------- 35 files changed, 121 insertions(+), 129 deletions(-) delete mode 100644 .changeset/mcp-server-starts-without-wizard.md delete mode 100644 .changeset/public-token-expiration-seconds.md delete mode 100644 .changeset/report-health.md delete mode 100644 .changeset/silent-build-logs.md delete mode 100644 .changeset/target-cli-notifications.md delete mode 100644 .server-changes/agent-detail-metrics-layout.md delete mode 100644 .server-changes/fix-invite-role-change-for-existing-members.md delete mode 100644 .server-changes/internal-api-origin-flag.md delete mode 100644 .server-changes/paginate-concurrency-keys-table.md delete mode 100644 .server-changes/queue-concurrency-percent-limits.md delete mode 100644 .server-changes/queue-metrics-dashboard.md delete mode 100644 .server-changes/reject-queue-limit-above-env-limit.md delete mode 100644 .server-changes/remove-unused-trace-sync-route.md delete mode 100644 .server-changes/sentry-request-attribution.md delete mode 100644 .server-changes/tsql-query-error-log-levels.md diff --git a/.changeset/mcp-server-starts-without-wizard.md b/.changeset/mcp-server-starts-without-wizard.md deleted file mode 100644 index 72a1a8bc659..00000000000 --- a/.changeset/mcp-server-starts-without-wizard.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"trigger.dev": patch ---- - -`trigger mcp` now always starts the MCP server, and the interactive install wizard has moved behind `trigger mcp --install`. Previously the wizard opened whenever stdout was a terminal, so any MCP host that spawns the command over a pseudo-terminal waited on a server that never started and eventually timed out. diff --git a/.changeset/public-token-expiration-seconds.md b/.changeset/public-token-expiration-seconds.md deleted file mode 100644 index 5eba6ba689b..00000000000 --- a/.changeset/public-token-expiration-seconds.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@trigger.dev/sdk": patch ---- - -Correct the `expirationTime` docs on `auth.createPublicToken` and the trigger-token helpers: a number is a Unix timestamp in seconds, not milliseconds. diff --git a/.changeset/report-health.md b/.changeset/report-health.md deleted file mode 100644 index 51f06733c7f..00000000000 --- a/.changeset/report-health.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -"@trigger.dev/core": patch -"trigger.dev": patch ---- - -Ask whether an environment is healthy and get an answer instead of a wall of charts. `trigger report health` returns a verdict on three questions: is work flowing, are the runs that start succeeding, and is the telemetry fresh enough to trust either answer. When something looks wrong it names the most likely cause and a next action. - -```bash -npx trigger.dev@latest report health --env prod --period 24h -``` - -The verdict is computed server side, so the CLI, the new `get_report` MCP tool, and `GET /api/v1/reports/health` all return the same text with the same sparklines. In MCP hosts that support prompts, `report` is also available as a slash command. diff --git a/.changeset/silent-build-logs.md b/.changeset/silent-build-logs.md deleted file mode 100644 index 17c3b9ab0f3..00000000000 --- a/.changeset/silent-build-logs.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"trigger.dev": patch ---- - -Prevent build debug logs from including environment variable values. diff --git a/.changeset/target-cli-notifications.md b/.changeset/target-cli-notifications.md deleted file mode 100644 index bdb6f7240c0..00000000000 --- a/.changeset/target-cli-notifications.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"trigger.dev": patch ---- - -Send the running CLI version when checking for platform notifications so notices can be limited to compatible CLI releases. diff --git a/.server-changes/agent-detail-metrics-layout.md b/.server-changes/agent-detail-metrics-layout.md deleted file mode 100644 index 1db02286f09..00000000000 --- a/.server-changes/agent-detail-metrics-layout.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: improvement ---- - -The agent detail page now uses the same layout as the other metrics pages: filters stay pinned at the top, the activity charts sit in a row beneath them, and the tabs and table scroll with the page, with the agent config in a resizable panel on the right. diff --git a/.server-changes/fix-invite-role-change-for-existing-members.md b/.server-changes/fix-invite-role-change-for-existing-members.md deleted file mode 100644 index ad4fb18b622..00000000000 --- a/.server-changes/fix-invite-role-change-for-existing-members.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -Accepting an old invitation could change the role of someone who was already in the organization. An invitation now leaves an existing member's role untouched, people who are already in an organization are no longer sent invitations to it, and the invite form now says which addresses it skipped instead of failing with an unhelpful error. diff --git a/.server-changes/internal-api-origin-flag.md b/.server-changes/internal-api-origin-flag.md deleted file mode 100644 index 48b26ba8536..00000000000 --- a/.server-changes/internal-api-origin-flag.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: improvement ---- - -Self-hosted instances can now serve deployed runs' API traffic from a different origin than the public one, per organization, via the `INTERNAL_API_ORIGIN` environment variable and a feature flag. diff --git a/.server-changes/paginate-concurrency-keys-table.md b/.server-changes/paginate-concurrency-keys-table.md deleted file mode 100644 index 3f9c610362d..00000000000 --- a/.server-changes/paginate-concurrency-keys-table.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: improvement ---- - -A queue's concurrency keys are now paged and searchable, so a queue with thousands of keys shows all of them instead of only the busiest 50. diff --git a/.server-changes/queue-concurrency-percent-limits.md b/.server-changes/queue-concurrency-percent-limits.md deleted file mode 100644 index 5e4d70c5433..00000000000 --- a/.server-changes/queue-concurrency-percent-limits.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: feature ---- - -A queue's concurrency limit can now be set as a percentage of your environment limit, from the dashboard or the API. Percentage limits track the environment limit, so raising or lowering it re-divides capacity without you editing every queue. diff --git a/.server-changes/queue-metrics-dashboard.md b/.server-changes/queue-metrics-dashboard.md deleted file mode 100644 index 4892040a639..00000000000 --- a/.server-changes/queue-metrics-dashboard.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: feature ---- - -See how every queue in an environment is doing: backlog, throughput, concurrency against its limit, when it was throttled, and how long runs waited before starting. Each queue also gets its own page, which breaks the same numbers down per concurrency key so you can tell whether one key is starving the rest. diff --git a/.server-changes/reject-queue-limit-above-env-limit.md b/.server-changes/reject-queue-limit-above-env-limit.md deleted file mode 100644 index dcd1dad89f8..00000000000 --- a/.server-changes/reject-queue-limit-above-env-limit.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: breaking ---- - -Setting a queue concurrency limit higher than your environment limit is now rejected with an error instead of being silently reduced to the environment limit, so a queue never looks like it has more capacity than it can get. Limits already saved are unchanged. diff --git a/.server-changes/remove-unused-trace-sync-route.md b/.server-changes/remove-unused-trace-sync-route.md deleted file mode 100644 index b2e857405a0..00000000000 --- a/.server-changes/remove-unused-trace-sync-route.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -Removed an unused internal dashboard sync route that is no longer part of any product surface. diff --git a/.server-changes/sentry-request-attribution.md b/.server-changes/sentry-request-attribution.md deleted file mode 100644 index d42abf78d09..00000000000 --- a/.server-changes/sentry-request-attribution.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -Fixed error reports being attributed to the wrong request when several requests were in flight at once. diff --git a/.server-changes/tsql-query-error-log-levels.md b/.server-changes/tsql-query-error-log-levels.md deleted file mode 100644 index 1aaedba3b76..00000000000 --- a/.server-changes/tsql-query-error-log-levels.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -Invalid queries sent to the query API are no longer treated as internal errors, and a query that does fail is now recorded together with the query text that produced it. diff --git a/hosting/k8s/helm/Chart.yaml b/hosting/k8s/helm/Chart.yaml index 69538c1a239..f1753ef7ebc 100644 --- a/hosting/k8s/helm/Chart.yaml +++ b/hosting/k8s/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: trigger description: The official Trigger.dev Helm chart type: application -version: 4.5.8 -appVersion: v4.5.8 +version: 4.5.9 +appVersion: v4.5.9 home: https://trigger.dev sources: - https://github.com/triggerdotdev/trigger.dev diff --git a/packages/build/CHANGELOG.md b/packages/build/CHANGELOG.md index 10f67297541..86758b44ec7 100644 --- a/packages/build/CHANGELOG.md +++ b/packages/build/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/build +## 4.5.9 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.9` + ## 4.5.8 ### Patch Changes diff --git a/packages/build/package.json b/packages/build/package.json index 39c7e5cd703..c55ffbb687e 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/build", - "version": "4.5.8", + "version": "4.5.9", "description": "trigger.dev build extensions", "license": "MIT", "publishConfig": { @@ -79,7 +79,7 @@ }, "dependencies": { "@prisma/config": "^6.10.0", - "@trigger.dev/core": "workspace:4.5.8", + "@trigger.dev/core": "workspace:4.5.9", "mlly": "^1.7.1", "pkg-types": "^1.1.3", "resolve": "^1.22.8", diff --git a/packages/cli-v3/CHANGELOG.md b/packages/cli-v3/CHANGELOG.md index c3821d48b18..2305a7b0fdb 100644 --- a/packages/cli-v3/CHANGELOG.md +++ b/packages/cli-v3/CHANGELOG.md @@ -1,5 +1,25 @@ # trigger.dev +## 4.5.9 + +### Patch Changes + +- `trigger mcp` now always starts the MCP server, and the interactive install wizard has moved behind `trigger mcp --install`. Previously the wizard opened whenever stdout was a terminal, so any MCP host that spawns the command over a pseudo-terminal waited on a server that never started and eventually timed out. ([#4131](https://github.com/triggerdotdev/trigger.dev/pull/4131)) +- Ask whether an environment is healthy and get an answer instead of a wall of charts. `trigger report health` returns a verdict on three questions: is work flowing, are the runs that start succeeding, and is the telemetry fresh enough to trust either answer. When something looks wrong it names the most likely cause and a next action. ([#4131](https://github.com/triggerdotdev/trigger.dev/pull/4131)) + + ```bash + npx trigger.dev@latest report health --env prod --period 24h + ``` + + The verdict is computed server side, so the CLI, the new `get_report` MCP tool, and `GET /api/v1/reports/health` all return the same text with the same sparklines. In MCP hosts that support prompts, `report` is also available as a slash command. + +- Prevent build debug logs from including environment variable values. ([#4420](https://github.com/triggerdotdev/trigger.dev/pull/4420)) +- Send the running CLI version when checking for platform notifications so notices can be limited to compatible CLI releases. ([#4407](https://github.com/triggerdotdev/trigger.dev/pull/4407)) +- Updated dependencies: + - `@trigger.dev/core@4.5.9` + - `@trigger.dev/build@4.5.9` + - `@trigger.dev/schema-to-json@4.5.9` + ## 4.5.8 ### Patch Changes diff --git a/packages/cli-v3/package.json b/packages/cli-v3/package.json index 9dba77e6af7..44c1ec0466d 100644 --- a/packages/cli-v3/package.json +++ b/packages/cli-v3/package.json @@ -1,6 +1,6 @@ { "name": "trigger.dev", - "version": "4.5.8", + "version": "4.5.9", "description": "A Command-Line Interface for Trigger.dev projects", "type": "module", "license": "MIT", @@ -96,9 +96,9 @@ "@opentelemetry/sdk-trace-node": "2.7.1", "@opentelemetry/semantic-conventions": "1.41.1", "@s2-dev/streamstore": "^0.22.10", - "@trigger.dev/build": "workspace:4.5.8", - "@trigger.dev/core": "workspace:4.5.8", - "@trigger.dev/schema-to-json": "workspace:4.5.8", + "@trigger.dev/build": "workspace:4.5.9", + "@trigger.dev/core": "workspace:4.5.9", + "@trigger.dev/schema-to-json": "workspace:4.5.9", "ansi-escapes": "^7.0.0", "braces": "^3.0.3", "c12": "^1.11.1", diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index 91222935306..b1a86d8356d 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -1,5 +1,17 @@ # internal-platform +## 4.5.9 + +### Patch Changes + +- Ask whether an environment is healthy and get an answer instead of a wall of charts. `trigger report health` returns a verdict on three questions: is work flowing, are the runs that start succeeding, and is the telemetry fresh enough to trust either answer. When something looks wrong it names the most likely cause and a next action. ([#4131](https://github.com/triggerdotdev/trigger.dev/pull/4131)) + + ```bash + npx trigger.dev@latest report health --env prod --period 24h + ``` + + The verdict is computed server side, so the CLI, the new `get_report` MCP tool, and `GET /api/v1/reports/health` all return the same text with the same sparklines. In MCP hosts that support prompts, `report` is also available as a slash command. + ## 4.5.8 ### Patch Changes diff --git a/packages/core/package.json b/packages/core/package.json index c99f83fe658..59cb1c76672 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/core", - "version": "4.5.8", + "version": "4.5.9", "description": "Core code used across the Trigger.dev SDK and platform", "license": "MIT", "publishConfig": { diff --git a/packages/python/CHANGELOG.md b/packages/python/CHANGELOG.md index ef31e38a05c..5ed18bb5054 100644 --- a/packages/python/CHANGELOG.md +++ b/packages/python/CHANGELOG.md @@ -1,5 +1,14 @@ # @trigger.dev/python +## 4.5.9 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/sdk@4.5.9` + - `@trigger.dev/core@4.5.9` + - `@trigger.dev/build@4.5.9` + ## 4.5.8 ### Patch Changes diff --git a/packages/python/package.json b/packages/python/package.json index 3b1fbb67823..b0c766b947a 100644 --- a/packages/python/package.json +++ b/packages/python/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/python", - "version": "4.5.8", + "version": "4.5.9", "description": "Python runtime and build extension for Trigger.dev", "license": "MIT", "publishConfig": { @@ -45,7 +45,7 @@ "check-exports": "attw --pack ." }, "dependencies": { - "@trigger.dev/core": "workspace:4.5.8", + "@trigger.dev/core": "workspace:4.5.9", "tinyexec": "^0.3.2" }, "devDependencies": { @@ -56,12 +56,12 @@ "tsx": "4.17.0", "esbuild": "^0.23.0", "@arethetypeswrong/cli": "^0.18.5", - "@trigger.dev/build": "workspace:4.5.8", - "@trigger.dev/sdk": "workspace:4.5.8" + "@trigger.dev/build": "workspace:4.5.9", + "@trigger.dev/sdk": "workspace:4.5.9" }, "peerDependencies": { - "@trigger.dev/sdk": "workspace:^4.5.8", - "@trigger.dev/build": "workspace:^4.5.8" + "@trigger.dev/sdk": "workspace:^4.5.9", + "@trigger.dev/build": "workspace:^4.5.9" }, "engines": { "node": ">=18.20.0" diff --git a/packages/react-hooks/CHANGELOG.md b/packages/react-hooks/CHANGELOG.md index 78140e5a18b..7f61711e1d1 100644 --- a/packages/react-hooks/CHANGELOG.md +++ b/packages/react-hooks/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/react-hooks +## 4.5.9 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.9` + ## 4.5.8 ### Patch Changes diff --git a/packages/react-hooks/package.json b/packages/react-hooks/package.json index 48a5b67a6b1..cc8bc3dd300 100644 --- a/packages/react-hooks/package.json +++ b/packages/react-hooks/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/react-hooks", - "version": "4.5.8", + "version": "4.5.9", "description": "trigger.dev react hooks", "license": "MIT", "publishConfig": { @@ -37,7 +37,7 @@ "check-exports": "attw --pack ." }, "dependencies": { - "@trigger.dev/core": "workspace:^4.5.8", + "@trigger.dev/core": "workspace:^4.5.9", "swr": "^2.2.5" }, "devDependencies": { diff --git a/packages/redis-worker/CHANGELOG.md b/packages/redis-worker/CHANGELOG.md index 1fbed6edc3e..a16a30bfda2 100644 --- a/packages/redis-worker/CHANGELOG.md +++ b/packages/redis-worker/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/redis-worker +## 4.5.9 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.9` + ## 4.5.8 ### Patch Changes diff --git a/packages/redis-worker/package.json b/packages/redis-worker/package.json index 7d2ff4dceea..7c014c23436 100644 --- a/packages/redis-worker/package.json +++ b/packages/redis-worker/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/redis-worker", - "version": "4.5.8", + "version": "4.5.9", "description": "Redis worker for trigger.dev", "license": "MIT", "publishConfig": { @@ -23,7 +23,7 @@ "test": "vitest --sequence.concurrent=false --no-file-parallelism" }, "dependencies": { - "@trigger.dev/core": "workspace:4.5.8", + "@trigger.dev/core": "workspace:4.5.9", "lodash.omit": "^4.5.0", "nanoid": "^5.0.7", "p-limit": "^6.2.0", diff --git a/packages/rsc/CHANGELOG.md b/packages/rsc/CHANGELOG.md index f30b7980f35..113bca6feb5 100644 --- a/packages/rsc/CHANGELOG.md +++ b/packages/rsc/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/rsc +## 4.5.9 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.9` + ## 4.5.8 ### Patch Changes diff --git a/packages/rsc/package.json b/packages/rsc/package.json index b216d219d8c..eb6396223dc 100644 --- a/packages/rsc/package.json +++ b/packages/rsc/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/rsc", - "version": "4.5.8", + "version": "4.5.9", "description": "trigger.dev rsc", "license": "MIT", "publishConfig": { @@ -37,14 +37,14 @@ "check-exports": "attw --pack ." }, "dependencies": { - "@trigger.dev/core": "workspace:^4.5.8", + "@trigger.dev/core": "workspace:^4.5.9", "mlly": "^1.7.1", "react": "19.0.0-rc.1", "react-dom": "19.0.0-rc.1" }, "devDependencies": { "@arethetypeswrong/cli": "^0.18.5", - "@trigger.dev/build": "workspace:^4.5.8", + "@trigger.dev/build": "workspace:^4.5.9", "@types/node": "^24.13.3", "@types/react": "*", "@types/react-dom": "*", diff --git a/packages/schema-to-json/CHANGELOG.md b/packages/schema-to-json/CHANGELOG.md index 6f5a7cc5bae..0b5f9548c03 100644 --- a/packages/schema-to-json/CHANGELOG.md +++ b/packages/schema-to-json/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/schema-to-json +## 4.5.9 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.9` + ## 4.5.8 ### Patch Changes diff --git a/packages/schema-to-json/package.json b/packages/schema-to-json/package.json index 66aa20c358f..0a590345abc 100644 --- a/packages/schema-to-json/package.json +++ b/packages/schema-to-json/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/schema-to-json", - "version": "4.5.8", + "version": "4.5.9", "description": "Convert various schema validation libraries to JSON Schema", "license": "MIT", "publishConfig": { diff --git a/packages/trigger-sdk/CHANGELOG.md b/packages/trigger-sdk/CHANGELOG.md index fa4b2ddb99f..9abce6e7ee5 100644 --- a/packages/trigger-sdk/CHANGELOG.md +++ b/packages/trigger-sdk/CHANGELOG.md @@ -1,5 +1,13 @@ # @trigger.dev/sdk +## 4.5.9 + +### Patch Changes + +- Correct the `expirationTime` docs on `auth.createPublicToken` and the trigger-token helpers: a number is a Unix timestamp in seconds, not milliseconds. ([#4388](https://github.com/triggerdotdev/trigger.dev/pull/4388)) +- Updated dependencies: + - `@trigger.dev/core@4.5.9` + ## 4.5.8 ### Patch Changes diff --git a/packages/trigger-sdk/package.json b/packages/trigger-sdk/package.json index 41b5591c8ad..64de5ea6d96 100644 --- a/packages/trigger-sdk/package.json +++ b/packages/trigger-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/sdk", - "version": "4.5.8", + "version": "4.5.9", "description": "trigger.dev Node.JS SDK", "license": "MIT", "publishConfig": { @@ -77,7 +77,7 @@ "dependencies": { "@opentelemetry/api": "1.9.1", "@opentelemetry/semantic-conventions": "1.41.1", - "@trigger.dev/core": "workspace:4.5.8", + "@trigger.dev/core": "workspace:4.5.9", "chalk": "^5.2.0", "cronstrue": "^2.21.0", "debug": "^4.3.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 766ab1a8eb4..2bb6eabbd1f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1450,7 +1450,7 @@ importers: specifier: ^6.10.0 version: 6.19.0(magicast@0.3.5) '@trigger.dev/core': - specifier: workspace:4.5.8 + specifier: workspace:4.5.9 version: link:../core mlly: specifier: ^1.7.1 @@ -1526,13 +1526,13 @@ importers: specifier: ^0.22.10 version: 0.22.10(supports-color@10.0.0) '@trigger.dev/build': - specifier: workspace:4.5.8 + specifier: workspace:4.5.9 version: link:../build '@trigger.dev/core': - specifier: workspace:4.5.8 + specifier: workspace:4.5.9 version: link:../core '@trigger.dev/schema-to-json': - specifier: workspace:4.5.8 + specifier: workspace:4.5.9 version: link:../schema-to-json ansi-escapes: specifier: ^7.0.0 @@ -1919,7 +1919,7 @@ importers: packages/python: dependencies: '@trigger.dev/core': - specifier: workspace:4.5.8 + specifier: workspace:4.5.9 version: link:../core tinyexec: specifier: ^0.3.2 @@ -1929,10 +1929,10 @@ importers: specifier: ^0.18.5 version: 0.18.5 '@trigger.dev/build': - specifier: workspace:4.5.8 + specifier: workspace:4.5.9 version: link:../build '@trigger.dev/sdk': - specifier: workspace:4.5.8 + specifier: workspace:4.5.9 version: link:../trigger-sdk '@types/node': specifier: 24.13.3 @@ -1956,7 +1956,7 @@ importers: packages/react-hooks: dependencies: '@trigger.dev/core': - specifier: workspace:^4.5.8 + specifier: workspace:^4.5.9 version: link:../core react: specifier: 18.3.1 @@ -1990,7 +1990,7 @@ importers: packages/redis-worker: dependencies: '@trigger.dev/core': - specifier: workspace:4.5.8 + specifier: workspace:4.5.9 version: link:../core cron-parser: specifier: ^4.9.0 @@ -2039,7 +2039,7 @@ importers: packages/rsc: dependencies: '@trigger.dev/core': - specifier: workspace:^4.5.8 + specifier: workspace:^4.5.9 version: link:../core mlly: specifier: ^1.7.1 @@ -2055,7 +2055,7 @@ importers: specifier: ^0.18.5 version: 0.18.5 '@trigger.dev/build': - specifier: workspace:^4.5.8 + specifier: workspace:^4.5.9 version: link:../build '@types/node': specifier: 24.13.3 @@ -2134,7 +2134,7 @@ importers: specifier: 1.41.1 version: 1.41.1 '@trigger.dev/core': - specifier: workspace:4.5.8 + specifier: workspace:4.5.9 version: link:../core chalk: specifier: ^5.2.0 From d90f06ba5e7024cfb757b095899415e61f30cf22 Mon Sep 17 00:00:00 2001 From: Iss <74388823+isshaddad@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:55:20 -0400 Subject: [PATCH 40/67] feat(webapp): migrate Plain to @team-plain/graphql + attribute support threads to org tenant (#4368) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Two changes, shipped together: 1. **SDK migration (TRI-12460).** `@team-plain/typescript-sdk` is deprecated. Move the webapp to its successors — `@team-plain/graphql` (client) and `@team-plain/ui-components` (`uiComponent` builder). Behaviour-preserving: the `PlainClient` customer upsert + thread creation move to the new `client.mutation.*({ input })` shape; the client now throws on failure, so `sendToPlain` wraps its calls and logs, staying best-effort. 2. **Org tenant attribution (TRI-12461).** When org context is available, `sendToPlain` now upserts a Plain tenant keyed by `externalId = org_id`, links the customer to it, and stamps the created thread with that tenant — so support threads become attributable to a Trigger.dev org. Wired into the four add-on quota requests and the plan-cancellation feedback (which already have org context). The tenant steps are isolated in their own try/catch and the thread's `tenantIdentifier` is gated on their success, so a tenant failure never blocks thread creation. ## Not affected - `customer.externalId` stays `User.id` — the customer cards + impersonation link are unchanged. - No ticket content leaves Plain. - Callers without a single org (e.g. the feedback widget) are unchanged — the org params are optional. ## Deploy prerequisite The webapp's Plain API key needs three **new** scopes for attribution to work (it already has `customer:create`, `customer:edit`, `thread:create`): - [x] `tenant:create` - [x] `tenant:edit` - [x] `customerTenantMembership:create` Until granted, nothing breaks — `sendToPlain` logs the forbidden error and creates the thread without attribution. ## Testing - `pnpm typecheck --filter webapp` passes; oxfmt + oxlint clean. - Ran the real `sendToPlain` end-to-end via a throwaway vitest harness against live Plain — confirmed the code path executes; the live write is gated only by the key scopes above. --- .server-changes/plain-graphql-migration.md | 6 + .../app/routes/api.v1.plain.customer-cards.ts | 2 +- apps/webapp/app/routes/resources.feedback.ts | 4 +- ...ces.orgs.$organizationSlug.select-plan.tsx | 4 +- apps/webapp/app/utils/plain.server.ts | 135 +++++++++++++----- .../v3/services/setBranchesAddOn.server.ts | 4 +- .../v3/services/setConcurrencyAddOn.server.ts | 4 +- .../v3/services/setSchedulesAddOn.server.ts | 4 +- .../app/v3/services/setSeatsAddOn.server.ts | 4 +- apps/webapp/package.json | 3 +- pnpm-lock.yaml | 40 ++++-- 11 files changed, 148 insertions(+), 62 deletions(-) create mode 100644 .server-changes/plain-graphql-migration.md diff --git a/.server-changes/plain-graphql-migration.md b/.server-changes/plain-graphql-migration.md new file mode 100644 index 00000000000..81468a91444 --- /dev/null +++ b/.server-changes/plain-graphql-migration.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +In-app feedback and add-on/quota requests are now recorded with more account context for the support team. diff --git a/apps/webapp/app/routes/api.v1.plain.customer-cards.ts b/apps/webapp/app/routes/api.v1.plain.customer-cards.ts index ad869c7a4f7..5e09fb08544 100644 --- a/apps/webapp/app/routes/api.v1.plain.customer-cards.ts +++ b/apps/webapp/app/routes/api.v1.plain.customer-cards.ts @@ -1,6 +1,6 @@ import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; import { timingSafeEqual } from "crypto"; -import { uiComponent } from "@team-plain/typescript-sdk"; +import { uiComponent } from "@team-plain/ui-components"; import { z } from "zod"; import { prisma } from "~/db.server"; import { env } from "~/env.server"; diff --git a/apps/webapp/app/routes/resources.feedback.ts b/apps/webapp/app/routes/resources.feedback.ts index bdb16ab7991..4ace262f0cb 100644 --- a/apps/webapp/app/routes/resources.feedback.ts +++ b/apps/webapp/app/routes/resources.feedback.ts @@ -1,13 +1,11 @@ import { parseWithZod } from "@conform-to/zod"; import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; -import { type PlainClient, uiComponent } from "@team-plain/typescript-sdk"; +import { uiComponent } from "@team-plain/ui-components"; import { z } from "zod"; import { redirectWithSuccessMessage } from "~/models/message.server"; import { requireUser } from "~/services/session.server"; import { sendToPlain } from "~/utils/plain.server"; -let _client: PlainClient | undefined; - export const feedbackTypes = { bug: { label: "Bug report", diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.select-plan.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.select-plan.tsx index 2c107c35b13..b72033098b7 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.select-plan.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.select-plan.tsx @@ -1,7 +1,7 @@ import { CheckIcon, XMarkIcon } from "@heroicons/react/20/solid"; import { ArrowDownCircleIcon, ArrowUpCircleIcon } from "@heroicons/react/24/outline"; import { Form, useLocation, useNavigation } from "@remix-run/react"; -import { uiComponent } from "@team-plain/typescript-sdk"; +import { uiComponent } from "@team-plain/ui-components"; import { type AddOnPricing, type FreePlanDefinition, @@ -95,6 +95,8 @@ export const action = dashboardAction( email: user.email, name: user.name ?? "", title: "Plan cancelation feedback", + organizationId: organization.id, + organizationName: organization.title, components: [ uiComponent.text({ text: `${user.name} (${user.email}) just canceled their plan.`, diff --git a/apps/webapp/app/utils/plain.server.ts b/apps/webapp/app/utils/plain.server.ts index c17b2f3f006..2dde84128e3 100644 --- a/apps/webapp/app/utils/plain.server.ts +++ b/apps/webapp/app/utils/plain.server.ts @@ -1,5 +1,5 @@ -import type { uiComponent } from "@team-plain/typescript-sdk"; -import { PlainClient } from "@team-plain/typescript-sdk"; +import { PlainClient } from "@team-plain/graphql"; +import type { uiComponent } from "@team-plain/ui-components"; import { env } from "~/env.server"; type Input = { @@ -9,9 +9,20 @@ type Input = { title: string; components: ReturnType<typeof uiComponent.text>[]; labelTypeIds?: string[]; + organizationId?: string; + organizationName?: string; }; -export async function sendToPlain({ userId, email, name, title, components, labelTypeIds }: Input) { +export async function sendToPlain({ + userId, + email, + name, + title, + components, + labelTypeIds, + organizationId, + organizationName, +}: Input) { if (!env.PLAIN_API_KEY) { return; } @@ -20,43 +31,93 @@ export async function sendToPlain({ userId, email, name, title, components, labe apiKey: env.PLAIN_API_KEY, }); - const upsertCustomerRes = await client.upsertCustomer({ - identifier: { - emailAddress: email, - }, - onCreate: { - externalId: userId, - fullName: name, - email: { - email: email, - isVerified: true, + // Best-effort support side-effect. Only transport/auth errors throw (caught below); business + // and validation failures come back in each mutation's `result.error`, so we check those inline. + try { + const upsertCustomerRes = await client.mutation.upsertCustomer({ + input: { + identifier: { + emailAddress: email, + }, + onCreate: { + externalId: userId, + fullName: name, + email: { + email: email, + isVerified: true, + }, + }, + onUpdate: { + externalId: { value: userId }, + fullName: { value: name }, + email: { + email: email, + isVerified: true, + }, + }, }, - }, - onUpdate: { - externalId: { value: userId }, - fullName: { value: name }, - email: { - email: email, - isVerified: true, - }, - }, - }); + }); - if (upsertCustomerRes.error) { - console.error("Failed to upsert customer in Plain", upsertCustomerRes.error); - return; - } + if (upsertCustomerRes.error || !upsertCustomerRes.customer?.id) { + console.error("Failed to upsert customer in Plain", upsertCustomerRes.error); + return; + } + const customerId = upsertCustomerRes.customer.id; - const createThreadRes = await client.createThread({ - customerIdentifier: { - customerId: upsertCustomerRes.data.customer.id, - }, - title: title, - components: components, - labelTypeIds, - }); + // Attribute the thread to the org so support data can be rolled up per org: the tenant is + // keyed by externalId = org_id. Isolated in its own try/catch, and the thread's + // tenantIdentifier is gated on success — so a tenant failure (e.g. an API key without + // tenant scope) downgrades to "no attribution" instead of dropping the thread. The + // customer's own externalId (User.id, used by the customer cards + impersonation link) is + // left untouched. + let tenantLinked = false; + if (organizationId) { + try { + const tenantRes = await client.mutation.upsertTenant({ + input: { + identifier: { externalId: organizationId }, + externalId: organizationId, + name: organizationName ?? organizationId, + }, + }); + // Only link + attribute if the tenant genuinely upserted — a mutation error comes back in + // `.error` (not thrown), and stamping the thread with a tenant that wasn't created would + // make createThread itself fail. + const membershipRes = tenantRes.error + ? undefined + : await client.mutation.addCustomerToTenants({ + input: { + customerIdentifier: { customerId }, + tenantIdentifiers: [{ externalId: organizationId }], + }, + }); + if (tenantRes.error) { + console.error("Failed to upsert Plain tenant", tenantRes.error); + } else if (membershipRes?.error) { + console.error("Failed to link Plain customer to tenant", membershipRes.error); + } else { + tenantLinked = true; + } + } catch (error) { + console.error("Failed to link Plain customer to org tenant", error); + } + } - if (createThreadRes.error) { - console.error("Failed to create thread in Plain", createThreadRes.error); + const threadRes = await client.mutation.createThread({ + input: { + customerIdentifier: { + customerId, + }, + title: title, + components: components, + labelTypeIds, + tenantIdentifier: tenantLinked ? { externalId: organizationId } : undefined, + }, + }); + if (threadRes.error) { + console.error("Failed to create Plain thread", threadRes.error); + } + } catch (error) { + console.error("Failed to send to Plain", error); } } diff --git a/apps/webapp/app/v3/services/setBranchesAddOn.server.ts b/apps/webapp/app/v3/services/setBranchesAddOn.server.ts index 55f4f5817d8..b14aebfbe9d 100644 --- a/apps/webapp/app/v3/services/setBranchesAddOn.server.ts +++ b/apps/webapp/app/v3/services/setBranchesAddOn.server.ts @@ -3,7 +3,7 @@ import { tryCatch } from "@trigger.dev/core/utils"; import { setBranchesAddOn } from "~/services/platform.v3.server"; import assertNever from "assert-never"; import { sendToPlain } from "~/utils/plain.server"; -import { uiComponent } from "@team-plain/typescript-sdk"; +import { uiComponent } from "@team-plain/ui-components"; type Input = { userId: string; @@ -74,6 +74,8 @@ export class SetBranchesAddOnService extends BaseService { email: user.email, name: user.name ?? user.displayName ?? user.email, title: `Preview branches quota request: ${amount}`, + organizationId, + organizationName: organization?.title, components: [ uiComponent.text({ text: `Org: ${organization?.title} (${organizationId})`, diff --git a/apps/webapp/app/v3/services/setConcurrencyAddOn.server.ts b/apps/webapp/app/v3/services/setConcurrencyAddOn.server.ts index 6daf970bbd6..da1f61b37fc 100644 --- a/apps/webapp/app/v3/services/setConcurrencyAddOn.server.ts +++ b/apps/webapp/app/v3/services/setConcurrencyAddOn.server.ts @@ -4,7 +4,7 @@ import { tryCatch } from "@trigger.dev/core"; import { setConcurrencyAddOn } from "~/services/platform.v3.server"; import assertNever from "assert-never"; import { sendToPlain } from "~/utils/plain.server"; -import { uiComponent } from "@team-plain/typescript-sdk"; +import { uiComponent } from "@team-plain/ui-components"; type Input = { userId: string; @@ -106,6 +106,8 @@ export class SetConcurrencyAddOnService extends BaseService { email: user.email, name: user.name ?? user.displayName ?? user.email, title: `Concurrency quota request: ${totalExtraConcurrency}`, + organizationId, + organizationName: organization?.title, components: [ uiComponent.text({ text: `Org: ${organization?.title} (${organizationId})`, diff --git a/apps/webapp/app/v3/services/setSchedulesAddOn.server.ts b/apps/webapp/app/v3/services/setSchedulesAddOn.server.ts index a96a6000765..42b3d120363 100644 --- a/apps/webapp/app/v3/services/setSchedulesAddOn.server.ts +++ b/apps/webapp/app/v3/services/setSchedulesAddOn.server.ts @@ -3,7 +3,7 @@ import { tryCatch } from "@trigger.dev/core/utils"; import { setSchedulesAddOn } from "~/services/platform.v3.server"; import assertNever from "assert-never"; import { sendToPlain } from "~/utils/plain.server"; -import { uiComponent } from "@team-plain/typescript-sdk"; +import { uiComponent } from "@team-plain/ui-components"; type Input = { userId: string; @@ -74,6 +74,8 @@ export class SetSchedulesAddOnService extends BaseService { email: user.email, name: user.name ?? user.displayName ?? user.email, title: `Schedules quota request: ${amount}`, + organizationId, + organizationName: organization?.title, components: [ uiComponent.text({ text: `Org: ${organization?.title} (${organizationId})`, diff --git a/apps/webapp/app/v3/services/setSeatsAddOn.server.ts b/apps/webapp/app/v3/services/setSeatsAddOn.server.ts index b9159c2ee7c..a761e7f9762 100644 --- a/apps/webapp/app/v3/services/setSeatsAddOn.server.ts +++ b/apps/webapp/app/v3/services/setSeatsAddOn.server.ts @@ -3,7 +3,7 @@ import { tryCatch } from "@trigger.dev/core/utils"; import { setSeatsAddOn } from "~/services/platform.v3.server"; import assertNever from "assert-never"; import { sendToPlain } from "~/utils/plain.server"; -import { uiComponent } from "@team-plain/typescript-sdk"; +import { uiComponent } from "@team-plain/ui-components"; type Input = { userId: string; @@ -74,6 +74,8 @@ export class SetSeatsAddOnService extends BaseService { email: user.email, name: user.name ?? user.displayName ?? user.email, title: `Seats quota request: ${amount}`, + organizationId, + organizationName: organization?.title, components: [ uiComponent.text({ text: `Org: ${organization?.title} (${organizationId})`, diff --git a/apps/webapp/package.json b/apps/webapp/package.json index c239f5339f8..06731a53e1c 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -113,7 +113,8 @@ "@tanstack/match-sorter-utils": "^8.19.4", "@tanstack/react-table": "^8.21.3", "@tanstack/react-virtual": "^3.0.4", - "@team-plain/typescript-sdk": "^3.5.0", + "@team-plain/graphql": "^1.3.0", + "@team-plain/ui-components": "^5.0.0", "@trigger.dev/companyicons": "^1.5.35", "@trigger.dev/core": "workspace:*", "@trigger.dev/database": "workspace:*", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2bb6eabbd1f..dce3aa6e582 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -467,9 +467,12 @@ importers: '@tanstack/react-virtual': specifier: ^3.0.4 version: 3.0.4(react-dom@18.3.1(react@18.3.1))(react@18.3.1) - '@team-plain/typescript-sdk': - specifier: ^3.5.0 - version: 3.5.0 + '@team-plain/graphql': + specifier: ^1.3.0 + version: 1.3.0 + '@team-plain/ui-components': + specifier: ^5.0.0 + version: 5.0.0(@team-plain/graphql@1.3.0) '@trigger.dev/companyicons': specifier: ^1.5.35 version: 1.5.35(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -7692,9 +7695,13 @@ packages: '@tanstack/virtual-core@3.0.0': resolution: {integrity: sha512-SYXOBTjJb05rXa2vl55TTwO40A6wKu0R5i1qQwhJYNDIqaIGF7D0HsLw+pJAyi2OvntlEIVusx3xtbbgSUi6zg==} - '@team-plain/typescript-sdk@3.5.0': - resolution: {integrity: sha512-9kweiSlYAN31VI7yzILGxdlZqsGJ+FmCEfXyEZ/0/i3r6vOwq45FDqtjadnQJVtFm+rf/8vCFRN+wEYMIEv6Aw==} - deprecated: This package is now deprecated. Please use @team-plain/graphql, @team-plain/webhooks and @team-plain/ui-components (https://github.com/team-plain/sdk) + '@team-plain/graphql@1.3.0': + resolution: {integrity: sha512-e+AZ9sIn4uliAyg7efcX6F0L4qOuwqL+6a+f5qe7Bfp7yXB5DU3vwECxHWN9YCD30MXnJ3oeS8Co1T5C5r0DUQ==} + + '@team-plain/ui-components@5.0.0': + resolution: {integrity: sha512-7/k8GggIGhdzkrZD3LAYWQZgJZi7VB3bQkpGbL67VrfoQGLYRumCMJ8jEvfLgX8n/tJ6Xn03u5etwLBBeb+Kqw==} + peerDependencies: + '@team-plain/graphql': 1.3.0 '@testcontainers/postgresql@11.14.0': resolution: {integrity: sha512-wYbJn8GRTj8qfqzfVubxioYWlHJU/ImIjuzPwyy9C5Qfo6g3GLduPZAj+BifvqTZjgT3gd4gFVLCPhBji7dc1w==} @@ -10775,8 +10782,8 @@ packages: grapheme-splitter@1.0.4: resolution: {integrity: sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==} - graphql@16.6.0: - resolution: {integrity: sha512-KPIBPDlW7NxrbT/eh4qPXz5FiFdL5UbaA0XUNz2Rp3Z3hqBSkbj0GVjwFDztsWVauZUWsbKHgMg++sk8UX0bkw==} + graphql@16.14.2: + resolution: {integrity: sha512-Chq1s4CY7jmh8gO2qvLIJyfCDIN+EHLFW/9iShnp1z8FjBQMoodWP1kDC36VAMXXIvAjj4ARa7ntfAV2BrjsbA==} engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} gunzip-maybe@1.4.2: @@ -18587,9 +18594,9 @@ snapshots: '@google-cloud/precise-date@4.0.0': {} - '@graphql-typed-document-node/core@3.2.0(graphql@16.6.0)': + '@graphql-typed-document-node/core@3.2.0(graphql@16.14.2)': dependencies: - graphql: 16.6.0 + graphql: 16.14.2 '@grpc/grpc-js@1.12.6': dependencies: @@ -22321,11 +22328,14 @@ snapshots: '@tanstack/virtual-core@3.0.0': {} - '@team-plain/typescript-sdk@3.5.0': + '@team-plain/graphql@1.3.0': dependencies: - '@graphql-typed-document-node/core': 3.2.0(graphql@16.6.0) - graphql: 16.6.0 - zod: 3.25.76 + '@graphql-typed-document-node/core': 3.2.0(graphql@16.14.2) + graphql: 16.14.2 + + '@team-plain/ui-components@5.0.0(@team-plain/graphql@1.3.0)': + dependencies: + '@team-plain/graphql': 1.3.0 '@testcontainers/postgresql@11.14.0': dependencies: @@ -25995,7 +26005,7 @@ snapshots: grapheme-splitter@1.0.4: {} - graphql@16.6.0: {} + graphql@16.14.2: {} gunzip-maybe@1.4.2: dependencies: From 4efe0a07c435e046874d0996469c4ed1f3b769b0 Mon Sep 17 00:00:00 2001 From: Oskar Otwinowski <oskar.otwinowski@gmail.com> Date: Thu, 30 Jul 2026 21:50:15 +0200 Subject: [PATCH 41/67] fix(webapp): create dev environments for SSO and Directory Sync members (#4426) Members added by SSO just-in-time provisioning or Directory Sync never got their per-member DEVELOPMENT environments - only invite acceptance and project creation created them. `trigger dev` returned "Environment not found" for those members and the dashboard had no dev view. ensureOrgMember now queues provisioning for every membership it settles, so both paths are covered and members missing environments are repaired on their next sync. Provisioning runs as a common-worker job to keep sign-in and directory webhooks off the per-project write loop. A failed enqueue surfaces for Directory Sync, whose worker retries the idempotent effect, and is swallowed for sign-in, where the next login enqueues again. Environment creation now tolerates a concurrent creator so the project-creation loop and the job cannot collide on the unique index. Also fixes environment resolution ignoring dev-environment ownership: a member without their own dev environment could be handed a colleague's and have it persisted as their dashboard preference. --- .../sso-directory-sync-dev-environments.md | 6 + apps/webapp/app/models/member.server.ts | 113 +++-- apps/webapp/app/models/orgMember.server.ts | 37 +- apps/webapp/app/models/organization.server.ts | 84 +++- apps/webapp/app/models/project.server.ts | 12 +- .../SelectBestEnvironmentPresenter.server.ts | 6 +- .../route.tsx | 25 +- .../route.tsx | 15 +- .../services/directorySyncEffects.server.ts | 36 +- .../services/memberDevEnvironments.server.ts | 105 +++++ .../app/utils/environmentAccess.test.ts | 54 +++ apps/webapp/app/utils/environmentAccess.ts | 20 + .../app/v3/accountsWebhookWorker.server.ts | 10 +- apps/webapp/app/v3/commonWorker.server.ts | 14 + .../test/directorySyncEffects.server.test.ts | 57 ++- .../test/envParamRoute.ownership.test.ts | 127 ++++++ apps/webapp/test/member.server.test.ts | 60 ++- .../test/memberDevEnvironments.server.test.ts | 411 ++++++++++++++++++ apps/webapp/test/orgMember.server.test.ts | 182 ++++++++ .../webapp/test/selectBestEnvironment.test.ts | 90 ++++ 20 files changed, 1390 insertions(+), 74 deletions(-) create mode 100644 .server-changes/sso-directory-sync-dev-environments.md create mode 100644 apps/webapp/app/services/memberDevEnvironments.server.ts create mode 100644 apps/webapp/app/utils/environmentAccess.test.ts create mode 100644 apps/webapp/app/utils/environmentAccess.ts create mode 100644 apps/webapp/test/envParamRoute.ownership.test.ts create mode 100644 apps/webapp/test/memberDevEnvironments.server.test.ts create mode 100644 apps/webapp/test/orgMember.server.test.ts create mode 100644 apps/webapp/test/selectBestEnvironment.test.ts diff --git a/.server-changes/sso-directory-sync-dev-environments.md b/.server-changes/sso-directory-sync-dev-environments.md new file mode 100644 index 00000000000..072482b8b19 --- /dev/null +++ b/.server-changes/sso-directory-sync-dev-environments.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Team members who join an organization through single sign-on or directory sync now get their own development environment for every project, so the dashboard's dev view and `trigger dev` become available to them without an admin having to set anything up. Members who were already missing one are repaired the next time their membership is synced from your identity provider. diff --git a/apps/webapp/app/models/member.server.ts b/apps/webapp/app/models/member.server.ts index 4167738413b..3be5f7ce09c 100644 --- a/apps/webapp/app/models/member.server.ts +++ b/apps/webapp/app/models/member.server.ts @@ -1,6 +1,10 @@ import type { Organization, OrgMember, Project } from "@trigger.dev/database"; +import { z } from "zod"; import { Prisma as PrismaNamespace, type Prisma, prisma } from "~/db.server"; -import { createEnvironment } from "./organization.server"; +import { + createDevelopmentEnvironmentForMember, + memberDevelopmentEnvironmentWhere, +} from "./organization.server"; import { customAlphabet } from "nanoid"; import { logger } from "~/services/logger.server"; import { getDefaultEnvironmentConcurrencyLimit } from "~/services/platform.v3.server"; @@ -13,6 +17,31 @@ export const INVITE_BLOCKED_DIRECTORY_MANAGED = export const ENV_SETUP_INCOMPLETE = "You joined the organization, but we couldn't finish setting up your development environments. Please try accepting the invite again, or contact support if this persists."; +/** How a membership came to exist. Also validates the queued job's payload. */ +export const MembershipSourceSchema = z.enum(["invite", "sso_jit", "directory_sync", "manual"]); + +export type MembershipSource = z.infer<typeof MembershipSourceSchema>; + +/** Thrown when provisioning fails partway through; the membership is still valid. */ +export class DevEnvironmentProvisioningError extends Error { + readonly logLevel = "warn" as const; + + constructor( + message: string, + readonly context: { + source: MembershipSource; + organizationId: string; + orgMemberId: string; + failedProjectId?: string; + createdProjectIds: string[]; + }, + options?: { cause?: unknown } + ) { + super(message, options); + this.name = "DevEnvironmentProvisioningError"; + } +} + export function isAcceptInviteFormError(error: unknown): error is Error { return ( error instanceof Error && @@ -186,7 +215,7 @@ export async function getUsersInvites({ email }: { email: string }) { }); } -async function getProjectsMissingMemberDevelopmentEnvironments({ +export async function getProjectsMissingMemberDevelopmentEnvironments({ memberId, organizationId, projects, @@ -201,10 +230,11 @@ async function getProjectsMissingMemberDevelopmentEnvironments({ const existingEnvs = await prisma.runtimeEnvironment.findMany({ where: { - orgMemberId: memberId, organizationId, - type: "DEVELOPMENT", - projectId: { in: projects.map((project) => project.id) }, + ...memberDevelopmentEnvironmentWhere({ + orgMemberId: memberId, + projectId: { in: projects.map((project) => project.id) }, + }), }, select: { projectId: true }, }); @@ -214,15 +244,15 @@ async function getProjectsMissingMemberDevelopmentEnvironments({ } export async function provisionMemberDevelopmentEnvironments({ + source, inviteId, - user, member, organization, projects, maximumConcurrencyLimit, }: { - inviteId: string; - user: { id: string; email: string }; + source: MembershipSource; + inviteId?: string; member: OrgMember; organization: Pick<Organization, "id" | "maximumConcurrencyLimit">; projects: Pick<Project, "id">[]; @@ -233,7 +263,7 @@ export async function provisionMemberDevelopmentEnvironments({ organizationId: organization.id, projects, }); - const projectIds = projects.map((project) => project.id); + const requestedProjectIds = projects.map((project) => project.id); const createdProjectIds: string[] = []; let failedProjectId: string | undefined; let failedProjectIndex: number | undefined; @@ -243,39 +273,74 @@ export async function provisionMemberDevelopmentEnvironments({ failedProjectId = project.id; failedProjectIndex = index; - await createEnvironment({ + const { created } = await createDevelopmentEnvironmentForMember({ organization, project, - type: "DEVELOPMENT", - // We set this true but no backfill (yet!?) so never used - // for dev environments - isBranchableEnvironment: true, member, maximumConcurrencyLimit, }); - createdProjectIds.push(project.id); + if (created) { + createdProjectIds.push(project.id); + } failedProjectId = undefined; failedProjectIndex = undefined; } } catch (error) { - logger.error("acceptInvite: development environment creation failed after membership created", { + const message = + "provisionMemberDevelopmentEnvironments: development environment creation failed after membership created"; + const context = { + source, inviteId, - userId: user.id, + userId: member.userId, organizationId: organization.id, orgMemberId: member.id, - projectIds, + requestedProjectIds, failedProjectId, failedProjectIndex, - totalProjects: projectsNeedingEnvs.length, + projectsNeedingEnvs: projectsNeedingEnvs.length, createdProjectIds, error: error instanceof Error ? { name: error.name, message: error.message, stack: error.stack } : String(error), - }); + }; + + if (source === "invite") { + logger.error(message, context); + } else { + logger.warn(message, context); + } - throw new Error(ENV_SETUP_INCOMPLETE); + throw new DevEnvironmentProvisioningError( + `Failed to create development environments for org member ${member.id}`, + { + source, + organizationId: organization.id, + orgMemberId: member.id, + failedProjectId, + createdProjectIds, + }, + { cause: error } + ); + } +} + +/** Provisions inline and surfaces a failure to the joiner as a retryable message. */ +async function provisionInviteDevelopmentEnvironments(args: { + inviteId: string; + member: OrgMember; + organization: Pick<Organization, "id" | "maximumConcurrencyLimit">; + projects: Pick<Project, "id">[]; + maximumConcurrencyLimit: number; +}) { + try { + await provisionMemberDevelopmentEnvironments({ source: "invite", ...args }); + } catch (error) { + if (error instanceof DevEnvironmentProvisioningError) { + throw new Error(ENV_SETUP_INCOMPLETE, { cause: error }); + } + throw error; } } @@ -375,9 +440,8 @@ async function tryRecoverIncompleteInviteAccept({ "DEVELOPMENT" ); - await provisionMemberDevelopmentEnvironments({ + await provisionInviteDevelopmentEnvironments({ inviteId, - user, member, organization: member.organization, projects: missingProjects, @@ -484,9 +548,8 @@ export async function acceptInvite({ } } - await provisionMemberDevelopmentEnvironments({ + await provisionInviteDevelopmentEnvironments({ inviteId, - user, member, organization: invite.organization, projects: invite.organization.projects, diff --git a/apps/webapp/app/models/orgMember.server.ts b/apps/webapp/app/models/orgMember.server.ts index d3ec2153d79..2bcd24b109f 100644 --- a/apps/webapp/app/models/orgMember.server.ts +++ b/apps/webapp/app/models/orgMember.server.ts @@ -1,5 +1,7 @@ import { Prisma, prisma } from "~/db.server"; +import type { MembershipSource } from "~/models/member.server"; import { logger } from "~/services/logger.server"; +import { enqueueMemberDevelopmentEnvironments } from "~/services/memberDevEnvironments.server"; import { rbac } from "~/services/rbac.server"; import { getValidPersonalAccessTokens, @@ -13,10 +15,15 @@ export type EnsureOrgMemberParams = { // value is an RBAC role id; when an RBAC plugin is installed it gets // attached after the OrgMember row is created. roleId: string | null; - source: "sso_jit" | "invite" | "manual" | "directory_sync"; + source: MembershipSource; }; -export type EnsureOrgMemberResult = { created: boolean; orgMemberId: string }; +export type EnsureOrgMemberResult = { + created: boolean; + orgMemberId: string; + /** False when provisioning could not be queued; the membership is still valid. */ + devEnvironmentsQueued: boolean; +}; // Completes a JIT role assignment for an ALREADY-existing membership whose // RBAC role never got applied. This is a no-op when a role is already @@ -82,7 +89,12 @@ export async function ensureOrgMember( if (roleId !== null) { await healMissingRoleAssignment({ userId, organizationId, roleId, source }); } - return { created: false, orgMemberId: existing.id }; + const { enqueued } = await enqueueMemberDevelopmentEnvironments({ + userId, + organizationId, + source, + }); + return { created: false, orgMemberId: existing.id, devEnvironmentsQueued: enqueued }; } // Two concurrent JIT/invite flows can both miss the findFirst above and @@ -106,7 +118,16 @@ export async function ensureOrgMember( select: { id: true }, }); if (existingAfterConflict) { - return { created: false, orgMemberId: existingAfterConflict.id }; + const { enqueued } = await enqueueMemberDevelopmentEnvironments({ + userId, + organizationId, + source, + }); + return { + created: false, + orgMemberId: existingAfterConflict.id, + devEnvironmentsQueued: enqueued, + }; } } throw error; @@ -134,7 +155,13 @@ export async function ensureOrgMember( } } - return { created: true, orgMemberId: member.id }; + const { enqueued } = await enqueueMemberDevelopmentEnvironments({ + userId, + organizationId, + source, + }); + + return { created: true, orgMemberId: member.id, devEnvironmentsQueued: enqueued }; } // Find-or-create a User for a directory-provisioned member. Directory Sync diff --git a/apps/webapp/app/models/organization.server.ts b/apps/webapp/app/models/organization.server.ts index 9f7fb382315..7022b60978b 100644 --- a/apps/webapp/app/models/organization.server.ts +++ b/apps/webapp/app/models/organization.server.ts @@ -10,7 +10,12 @@ import { tryCatch } from "@trigger.dev/core/utils"; import { customAlphabet } from "nanoid"; import { generate } from "random-words"; import slug from "slug"; -import { $replica, prisma, type PrismaClientOrTransaction } from "~/db.server"; +import { + $replica, + Prisma as PrismaNamespace, + prisma, + type PrismaClientOrTransaction, +} from "~/db.server"; import { env } from "~/env.server"; import { featuresForUrl } from "~/features.server"; import { createApiKeyForEnv, createPkApiKeyForEnv, envSlug } from "./api-key.server"; @@ -224,6 +229,83 @@ export async function createEnvironment({ return environment; } +/** + * A member's root development environment for a project, never a branch under + * it. Not keyed on slug, so a legacy root with another slug still matches. + */ +export function memberDevelopmentEnvironmentWhere({ + projectId, + orgMemberId, +}: { + projectId?: string | { in: string[] }; + orgMemberId: string; +}): Prisma.RuntimeEnvironmentWhereInput { + return { + ...(projectId === undefined ? {} : { projectId }), + orgMemberId, + type: "DEVELOPMENT", + parentEnvironmentId: null, + }; +} + +/** + * Create a member's development environment, reporting `created: false` when a + * concurrent writer already made it. Any other conflict still throws. + * + * Not transaction-aware: a unique violation aborts an enclosing transaction, so + * the read that confirms the concurrent row has to run outside one. + */ +export async function createDevelopmentEnvironmentForMember({ + organization, + project, + member, + maximumConcurrencyLimit, +}: { + organization: Pick<Organization, "id" | "maximumConcurrencyLimit">; + project: Pick<Project, "id">; + member: OrgMember; + maximumConcurrencyLimit?: number; +}): Promise<{ created: boolean }> { + try { + await createEnvironment({ + organization, + project, + type: "DEVELOPMENT", + isBranchableEnvironment: true, + member, + maximumConcurrencyLimit, + }); + return { created: true }; + } catch (error) { + if ( + !(error instanceof PrismaNamespace.PrismaClientKnownRequestError) || + error.code !== "P2002" + ) { + throw error; + } + + const existing = await prisma.runtimeEnvironment.findFirst({ + where: memberDevelopmentEnvironmentWhere({ + projectId: project.id, + orgMemberId: member.id, + }), + select: { id: true }, + }); + + if (!existing) { + throw error; + } + + logger.debug("Development environment already created by a concurrent writer", { + organizationId: organization.id, + projectId: project.id, + orgMemberId: member.id, + }); + + return { created: false }; + } +} + function createShortcode() { return generate({ exactly: 2 }); } diff --git a/apps/webapp/app/models/project.server.ts b/apps/webapp/app/models/project.server.ts index b579fcd2e22..2ed317fe879 100644 --- a/apps/webapp/app/models/project.server.ts +++ b/apps/webapp/app/models/project.server.ts @@ -4,7 +4,11 @@ import slug from "slug"; import { $replica, prisma } from "~/db.server"; import { projectCreated } from "~/services/projectCreated.server"; import { ServiceValidationError } from "~/v3/services/common.server"; -import { type Organization, createEnvironment } from "./organization.server"; +import { + type Organization, + createDevelopmentEnvironmentForMember, + createEnvironment, +} from "./organization.server"; export type { Project } from "@trigger.dev/database"; const externalRefGenerator = customAlphabet("abcdefghijklmnopqrstuvwxyz", 20); @@ -129,13 +133,9 @@ export async function createProject( }); for (const member of project.organization.members) { - await createEnvironment({ + await createDevelopmentEnvironmentForMember({ organization, project, - type: "DEVELOPMENT", - // We set this true but no backfill (yet!?) so never used - // for dev environments - isBranchableEnvironment: true, member, }); } diff --git a/apps/webapp/app/presenters/SelectBestEnvironmentPresenter.server.ts b/apps/webapp/app/presenters/SelectBestEnvironmentPresenter.server.ts index 1273c668c42..13d86911927 100644 --- a/apps/webapp/app/presenters/SelectBestEnvironmentPresenter.server.ts +++ b/apps/webapp/app/presenters/SelectBestEnvironmentPresenter.server.ts @@ -6,6 +6,7 @@ import { import { prisma } from "~/db.server"; import { logger } from "~/services/logger.server"; import { type UserFromSession } from "~/services/session.server"; +import { selectAccessibleEnvironment } from "~/utils/environmentAccess"; export type MinimumEnvironment = Pick<RuntimeEnvironment, "id" | "type" | "slug" | "paused"> & { orgMember: null | { @@ -154,7 +155,10 @@ export class SelectBestEnvironmentPresenter { const currentEnvironmentId: string | undefined = user.dashboardPreferences.projects[projectId]?.currentEnvironment.id; - const currentEnvironment = environments.find((env) => env.id === currentEnvironmentId); + const currentEnvironment = selectAccessibleEnvironment( + environments.filter((env) => env.id === currentEnvironmentId), + user.id + ); if (currentEnvironment) { return currentEnvironment; } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam/route.tsx index d22883caa12..c09238b6d3f 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam/route.tsx @@ -7,6 +7,7 @@ import { updateCurrentProjectEnvironmentId } from "~/services/dashboardPreferenc import { logger } from "~/services/logger.server"; import { requireUser } from "~/services/session.server"; import { tenantContext } from "~/services/tenantContext.server"; +import { selectAccessibleEnvironment } from "~/utils/environmentAccess"; import { EnvironmentParamSchema, v3ProjectPath } from "~/utils/pathBuilder"; import { canAccessDashboardAgent } from "~/v3/canAccessDashboardAgent.server"; @@ -52,28 +53,14 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { } const environments = project.environments.filter((env) => env.slug === envParam); - if (environments.length === 0) { + const environment = selectAccessibleEnvironment(environments, user.id); + + if (!environment) { return redirect(v3ProjectPath({ slug: organizationSlug }, { slug: projectParam })); } - let environmentId: string | undefined = undefined; - let environmentType: "DEVELOPMENT" | "PREVIEW" | "STAGING" | "PRODUCTION" | undefined; - - if (environments.length > 1) { - const bestEnvironment = environments.find((env) => env.orgMember?.userId === user.id); - if (!bestEnvironment) { - throw new Response("Environment not Found", { - status: 404, - statusText: "Environment not found", - }); - } - - environmentId = bestEnvironment.id; - environmentType = bestEnvironment.type; - } else { - environmentId = environments[0].id; - environmentType = environments[0].type; - } + const environmentId = environment.id; + const environmentType = environment.type; // userId is enriched higher up in `_app/route.tsx`; only stamp tenant fields here. tenantContext.enrich({ diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.sso/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.sso/route.tsx index 46abc2d4f2b..3d88220c4c2 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.sso/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.sso/route.tsx @@ -42,6 +42,7 @@ import { ssoController } from "~/services/sso.server"; import { getSsoEntitlement } from "~/services/platform.v3.server"; import type { DirectorySyncEffect, DirectorySyncStatus, Role } from "@trigger.dev/plugins"; import { applyDirectorySyncEffects } from "~/services/directorySyncEffects.server"; +import { logger } from "~/services/logger.server"; import { flag } from "~/v3/featureFlags.server"; import { FEATURE_FLAG } from "~/v3/featureFlags"; import { dashboardAction, dashboardLoader } from "~/services/routeBuilders/dashboardBuilder"; @@ -328,7 +329,19 @@ export const action = dashboardAction( effects.push(...result.value.effects); } if (effects.length > 0) { - await applyDirectorySyncEffects(effects); + // The save itself has already been persisted, so a provisioning + // enqueue that failed must not turn it into an error page. There is no + // retry on this path; the members are repaired on their next sync. + const { unqueuedUserIds } = await applyDirectorySyncEffects(effects); + if (unqueuedUserIds.length > 0) { + logger.warn( + "directorySync: could not queue development environments after settings save", + { + organizationId: orgId, + userIds: unqueuedUserIds, + } + ); + } } return redirect(`/orgs/${params.organizationSlug}/settings/sso`); } diff --git a/apps/webapp/app/services/directorySyncEffects.server.ts b/apps/webapp/app/services/directorySyncEffects.server.ts index 555bb6f98ab..9ede3897a05 100644 --- a/apps/webapp/app/services/directorySyncEffects.server.ts +++ b/apps/webapp/app/services/directorySyncEffects.server.ts @@ -67,7 +67,8 @@ async function notifyLastOwnerProtected(userId: string, organizationId: string): } } -async function applyEffect(effect: DirectorySyncEffect): Promise<void> { +/** Applies one effect, returning a user id that still needs provisioning queued. */ +async function applyEffect(effect: DirectorySyncEffect): Promise<string | null> { switch (effect.kind) { case "provision": { const userId = @@ -80,7 +81,7 @@ async function applyEffect(effect: DirectorySyncEffect): Promise<void> { }) ).userId; - await ensureOrgMember({ + const membership = await ensureOrgMember({ userId, organizationId: effect.organizationId, roleId: effect.roleId, @@ -111,7 +112,8 @@ async function applyEffect(effect: DirectorySyncEffect): Promise<void> { } } } - return; + + return membership.devEnvironmentsQueued ? null : userId; } case "set_role": { const result = await rbac.setUserRole({ @@ -127,11 +129,11 @@ async function applyEffect(effect: DirectorySyncEffect): Promise<void> { userId: effect.userId, organizationId: effect.organizationId, }); - return; + return null; } throw retryableEffectError(`directorySync set_role failed: ${result.error}`); } - return; + return null; } case "deprovision": { const outcome = await removeOrgMemberForDirectory({ @@ -141,20 +143,33 @@ async function applyEffect(effect: DirectorySyncEffect): Promise<void> { if (!outcome.removed && outcome.reason === "last_owner_protected") { await notifyLastOwnerProtected(effect.userId, effect.organizationId); } - return; + return null; } } } +/** Raised for a batch whose provisioning could not be queued, when the caller can retry. */ +export function unqueuedProvisioningError(userIds: string[]): Error { + return retryableEffectError( + `directorySync could not queue development environments for users ${userIds.join(", ")}` + ); +} + /** * Applies membership effects, skipping any org that isn't entitled to SSO. * * An unreadable entitlement throws rather than skipping: effects are * idempotent and the worker retries, so retrying is lossless where dropping * would silently lose a directory change. + * + * Reports memberships whose provisioning could not be queued rather than + * throwing, so a caller with no retry can still finish successfully. */ -export async function applyDirectorySyncEffects(effects: DirectorySyncEffect[]): Promise<void> { +export async function applyDirectorySyncEffects( + effects: DirectorySyncEffect[] +): Promise<{ unqueuedUserIds: string[] }> { const entitlements = new Map<string, SsoEntitlement>(); + const unqueuedUserIds: string[] = []; for (const effect of effects) { let entitlement = entitlements.get(effect.organizationId); @@ -177,6 +192,11 @@ export async function applyDirectorySyncEffects(effects: DirectorySyncEffect[]): continue; } - await applyEffect(effect); + const unqueuedUserId = await applyEffect(effect); + if (unqueuedUserId) { + unqueuedUserIds.push(unqueuedUserId); + } } + + return { unqueuedUserIds }; } diff --git a/apps/webapp/app/services/memberDevEnvironments.server.ts b/apps/webapp/app/services/memberDevEnvironments.server.ts new file mode 100644 index 00000000000..2c9dfc11a57 --- /dev/null +++ b/apps/webapp/app/services/memberDevEnvironments.server.ts @@ -0,0 +1,105 @@ +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { + getProjectsMissingMemberDevelopmentEnvironments, + MembershipSourceSchema, + provisionMemberDevelopmentEnvironments, + type MembershipSource, +} from "~/models/member.server"; +import { logger } from "~/services/logger.server"; +import { getDefaultEnvironmentConcurrencyLimit } from "~/services/platform.v3.server"; + +export const MembershipDevEnvironmentsSchema = z.object({ + userId: z.string(), + organizationId: z.string(), + source: MembershipSourceSchema, +}); + +export type MembershipDevEnvironments = z.infer<typeof MembershipDevEnvironmentsSchema>; + +/** + * Create the member's missing development environments, one per active project. + * Idempotent, so it is safe to re-run and to retry after a partial failure. + */ +export async function provisionDevEnvironmentsForMembership({ + userId, + organizationId, + source, +}: MembershipDevEnvironments): Promise<void> { + const member = await prisma.orgMember.findFirst({ + where: { + userId, + organizationId, + organization: { deletedAt: null }, + }, + include: { + organization: { + include: { + projects: { where: { deletedAt: null }, select: { id: true } }, + }, + }, + }, + }); + + if (!member) { + logger.info("provisionDevEnvironmentsForMembership: no membership found", { + userId, + organizationId, + source, + }); + return; + } + + const projectsNeedingEnvs = await getProjectsMissingMemberDevelopmentEnvironments({ + memberId: member.id, + organizationId, + projects: member.organization.projects, + }); + + if (projectsNeedingEnvs.length === 0) { + return; + } + + const maximumConcurrencyLimit = await getDefaultEnvironmentConcurrencyLimit( + organizationId, + "DEVELOPMENT" + ); + + await provisionMemberDevelopmentEnvironments({ + source, + member, + organization: member.organization, + projects: projectsNeedingEnvs, + maximumConcurrencyLimit, + }); +} + +/** + * Queue provisioning, deduped per membership. Never throws: callers decide what + * `enqueued: false` means for them. + */ +export async function enqueueMemberDevelopmentEnvironments(payload: { + userId: string; + organizationId: string; + source: MembershipSource; +}): Promise<{ enqueued: boolean }> { + try { + // Lazy: a static import would close a module cycle. + const { commonWorker } = await import("~/v3/commonWorker.server"); + + await commonWorker.enqueueOnce({ + id: `membership:devEnvs:${payload.organizationId}:${payload.userId}`, + job: "membership.provisionDevEnvironments", + payload, + }); + + return { enqueued: true }; + } catch (error) { + logger.error("Failed to enqueue member development environment provisioning", { + ...payload, + error: error instanceof Error ? error.message : String(error), + }); + + return { enqueued: false }; + } +} diff --git a/apps/webapp/app/utils/environmentAccess.test.ts b/apps/webapp/app/utils/environmentAccess.test.ts new file mode 100644 index 00000000000..5e5b0f5a7e5 --- /dev/null +++ b/apps/webapp/app/utils/environmentAccess.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest"; +import { selectAccessibleEnvironment } from "./environmentAccess"; + +const USER_ID = "user_owner"; +const OTHER_USER_ID = "user_other"; + +function devEnvironment(id: string, userId: string | null) { + return { + id, + type: "DEVELOPMENT" as const, + orgMember: userId === null ? null : { userId }, + }; +} + +function prodEnvironment(id: string) { + return { id, type: "PRODUCTION" as const, orgMember: null }; +} + +describe("selectAccessibleEnvironment", () => { + it("returns the caller's own development environment", () => { + const own = devEnvironment("env_own", USER_ID); + + expect(selectAccessibleEnvironment([own], USER_ID)).toBe(own); + }); + + it("skips a development environment belonging to another member", () => { + const theirs = devEnvironment("env_theirs", OTHER_USER_ID); + + expect(selectAccessibleEnvironment([theirs], USER_ID)).toBeUndefined(); + }); + + it("picks the caller's development environment past another member's", () => { + const theirs = devEnvironment("env_theirs", OTHER_USER_ID); + const own = devEnvironment("env_own", USER_ID); + + expect(selectAccessibleEnvironment([theirs, own], USER_ID)).toBe(own); + }); + + it("skips a development environment whose member has been deleted", () => { + const orphaned = devEnvironment("env_orphaned", null); + + expect(selectAccessibleEnvironment([orphaned], USER_ID)).toBeUndefined(); + }); + + it("returns a shared environment regardless of who owns it", () => { + const prod = prodEnvironment("env_prod"); + + expect(selectAccessibleEnvironment([prod], USER_ID)).toBe(prod); + }); + + it("returns undefined when there are no environments", () => { + expect(selectAccessibleEnvironment([], USER_ID)).toBeUndefined(); + }); +}); diff --git a/apps/webapp/app/utils/environmentAccess.ts b/apps/webapp/app/utils/environmentAccess.ts new file mode 100644 index 00000000000..d68113854f5 --- /dev/null +++ b/apps/webapp/app/utils/environmentAccess.ts @@ -0,0 +1,20 @@ +import { type RuntimeEnvironmentType } from "@trigger.dev/database"; + +type AccessibleEnvironmentInput = { + type: RuntimeEnvironmentType; + orgMember: { userId: string } | null; +}; + +/** + * The first environment the user may actually use. Development environments are + * per-member, so resolving one by slug or stored id alone can return another + * member's; every other type is shared by the project. + */ +export function selectAccessibleEnvironment<T extends AccessibleEnvironmentInput>( + environments: T[], + userId: string +): T | undefined { + return environments.find( + (environment) => environment.type !== "DEVELOPMENT" || environment.orgMember?.userId === userId + ); +} diff --git a/apps/webapp/app/v3/accountsWebhookWorker.server.ts b/apps/webapp/app/v3/accountsWebhookWorker.server.ts index cdd3ab50d34..fb8fb88b5ac 100644 --- a/apps/webapp/app/v3/accountsWebhookWorker.server.ts +++ b/apps/webapp/app/v3/accountsWebhookWorker.server.ts @@ -4,7 +4,10 @@ import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { singleton } from "~/utils/singleton"; import { ssoController } from "~/services/sso.server"; -import { applyDirectorySyncEffects } from "~/services/directorySyncEffects.server"; +import { + applyDirectorySyncEffects, + unqueuedProvisioningError, +} from "~/services/directorySyncEffects.server"; // Dedicated worker for inbound account-management webhooks. The webhook // proxy route verifies the signature via the plugin and enqueues the @@ -76,7 +79,10 @@ function initializeWorker() { // Directory-sync events return membership effects to apply against // public.* tables (the plugin never writes those). A throw here // bubbles to the worker for retry; effects are idempotent. - await applyDirectorySyncEffects(result.value.effects); + const { unqueuedUserIds } = await applyDirectorySyncEffects(result.value.effects); + if (unqueuedUserIds.length > 0) { + throw unqueuedProvisioningError(unqueuedUserIds); + } }, }, }); diff --git a/apps/webapp/app/v3/commonWorker.server.ts b/apps/webapp/app/v3/commonWorker.server.ts index 38f9ea94265..87093c36aae 100644 --- a/apps/webapp/app/v3/commonWorker.server.ts +++ b/apps/webapp/app/v3/commonWorker.server.ts @@ -12,6 +12,10 @@ import { runAttioWorkspaceSync, } from "~/services/attio.server"; import { logger } from "~/services/logger.server"; +import { + MembershipDevEnvironmentsSchema, + provisionDevEnvironmentsForMembership, +} from "~/services/memberDevEnvironments.server"; import { singleton } from "~/utils/singleton"; import { DeliverAlertService } from "./services/alerts/deliverAlert.server"; import { PerformDeploymentAlertsService } from "./services/alerts/performDeploymentAlerts.server"; @@ -58,6 +62,13 @@ function initializeWorker() { maxAttempts: 3, }, }, + "membership.provisionDevEnvironments": { + schema: MembershipDevEnvironmentsSchema, + visibilityTimeoutMs: 120_000, + retry: { + maxAttempts: 5, + }, + }, "v3.timeoutDeployment": { schema: z.object({ deploymentId: z.string(), @@ -155,6 +166,9 @@ function initializeWorker() { "attio.syncUser": async ({ payload }) => { await runAttioUserSync(payload); }, + "membership.provisionDevEnvironments": async ({ payload }) => { + await provisionDevEnvironmentsForMembership(payload); + }, "v3.timeoutDeployment": async ({ payload }) => { const service = new TimeoutDeploymentService(); await service.call(payload.deploymentId, payload.fromStatus, payload.errorMessage); diff --git a/apps/webapp/test/directorySyncEffects.server.test.ts b/apps/webapp/test/directorySyncEffects.server.test.ts index e8f9bb933cc..0d884c06355 100644 --- a/apps/webapp/test/directorySyncEffects.server.test.ts +++ b/apps/webapp/test/directorySyncEffects.server.test.ts @@ -50,7 +50,11 @@ function deprovision(organizationId: string): DirectorySyncEffect { describe("applyDirectorySyncEffects — SSO entitlement gate", () => { beforeEach(() => { vi.clearAllMocks(); - ensureOrgMember.mockResolvedValue(undefined); + ensureOrgMember.mockResolvedValue({ + created: true, + orgMemberId: "member_1", + devEnvironmentsQueued: true, + }); removeOrgMemberForDirectory.mockResolvedValue({ removed: true }); setUserRole.mockResolvedValue({ ok: true }); }); @@ -127,4 +131,55 @@ describe("applyDirectorySyncEffects — SSO entitlement gate", () => { expect.objectContaining({ organizationId: ENTITLED_ORG }) ); }); + + it("applies every effect in the batch and reports the unqueued members", async () => { + getSsoEntitlement.mockResolvedValue("entitled"); + ensureOrgMember + .mockResolvedValueOnce({ + created: true, + orgMemberId: "member_1", + devEnvironmentsQueued: false, + }) + .mockResolvedValueOnce({ + created: true, + orgMemberId: "member_2", + devEnvironmentsQueued: true, + }); + + const { unqueuedUserIds } = await applyDirectorySyncEffects([ + provision(ENTITLED_ORG, "a@acme.com"), + provision(ENTITLED_ORG, "b@acme.com"), + deprovision(ENTITLED_ORG), + ]); + + expect(unqueuedUserIds).toEqual(["user_1"]); + expect(ensureOrgMember).toHaveBeenCalledTimes(2); + expect(removeOrgMemberForDirectory).toHaveBeenCalledTimes(1); + }); + + it("applies the directory role even when provisioning could not be queued", async () => { + getSsoEntitlement.mockResolvedValue("entitled"); + ensureOrgMember.mockResolvedValue({ + created: false, + orgMemberId: "member_1", + devEnvironmentsQueued: false, + }); + + const effect = { ...provision(ENTITLED_ORG), roleId: "role_restricted" }; + + const { unqueuedUserIds } = await applyDirectorySyncEffects([effect]); + + expect(unqueuedUserIds).toEqual(["user_1"]); + expect(setUserRole).toHaveBeenCalledWith( + expect.objectContaining({ roleId: "role_restricted", organizationId: ENTITLED_ORG }) + ); + }); + + it("reports nothing to retry when every provision was queued", async () => { + getSsoEntitlement.mockResolvedValue("entitled"); + + const { unqueuedUserIds } = await applyDirectorySyncEffects([provision(ENTITLED_ORG)]); + + expect(unqueuedUserIds).toEqual([]); + }); }); diff --git a/apps/webapp/test/envParamRoute.ownership.test.ts b/apps/webapp/test/envParamRoute.ownership.test.ts new file mode 100644 index 00000000000..624ef711352 --- /dev/null +++ b/apps/webapp/test/envParamRoute.ownership.test.ts @@ -0,0 +1,127 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + user: { id: "user_me", admin: false, isImpersonating: false }, + project: null as unknown, + updatedEnvironmentIds: [] as string[], +})); + +vi.mock("~/db.server", () => ({ + prisma: { + project: { + findFirst: async () => mocks.project, + }, + }, + $replica: {}, +})); + +vi.mock("~/services/session.server", () => ({ + requireUser: async () => mocks.user, +})); + +vi.mock("~/services/dashboardPreferences.server", () => ({ + updateCurrentProjectEnvironmentId: async ({ environmentId }: { environmentId: string }) => { + mocks.updatedEnvironmentIds.push(environmentId); + }, +})); + +vi.mock("~/services/tenantContext.server", () => ({ + tenantContext: { enrich: () => {} }, +})); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => false, +})); + +vi.mock("~/services/logger.server", () => ({ + logger: { error: () => {}, warn: () => {}, info: () => {}, debug: () => {} }, +})); + +import { loader } from "~/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam/route"; + +const PARAMS = { + organizationSlug: "acme", + projectParam: "my-project", + envParam: "dev", +}; + +function devEnvironment(id: string, userId: string) { + return { id, type: "DEVELOPMENT" as const, slug: "dev", orgMember: { userId } }; +} + +function projectWith(environments: unknown[]) { + return { + id: "project_1", + externalRef: "proj_abc", + organization: { id: "org_1", featureFlags: {} }, + environments, + }; +} + +async function callLoader(params: typeof PARAMS = PARAMS) { + return (await loader({ + request: new Request( + `http://localhost:3030/orgs/acme/projects/my-project/env/${params.envParam}` + ), + params, + context: {}, + } as never)) as Response; +} + +describe("env.$envParam loader — development environment ownership", () => { + beforeEach(() => { + mocks.updatedEnvironmentIds.length = 0; + }); + + it("resolves the caller's own dev environment when several members have one", async () => { + mocks.project = projectWith([ + devEnvironment("env_colleague", "user_colleague"), + devEnvironment("env_mine", "user_me"), + ]); + + await callLoader(); + + expect(mocks.updatedEnvironmentIds).toEqual(["env_mine"]); + }); + + it("redirects rather than adopting the only other member's dev environment", async () => { + mocks.project = projectWith([devEnvironment("env_colleague", "user_colleague")]); + + const response = await callLoader(); + + expect(response.status).toBe(302); + expect(response.headers.get("location")).toBe("/orgs/acme/projects/my-project"); + expect(mocks.updatedEnvironmentIds).toEqual([]); + }); + + it("resolves a development branch to the caller's own, not a colleague's", async () => { + mocks.project = projectWith([ + { + id: "env_branch_theirs", + type: "DEVELOPMENT" as const, + slug: "dev-foo", + orgMember: { userId: "user_colleague" }, + }, + { + id: "env_branch_mine", + type: "DEVELOPMENT" as const, + slug: "dev-foo", + orgMember: { userId: "user_me" }, + }, + ]); + + await callLoader({ ...PARAMS, envParam: "dev-foo" }); + + expect(mocks.updatedEnvironmentIds).toEqual(["env_branch_mine"]); + }); + + it("still resolves shared environments that belong to no member", async () => { + mocks.project = projectWith([ + { id: "env_prod", type: "PRODUCTION" as const, slug: "dev", orgMember: null }, + ]); + + await callLoader(); + + expect(mocks.updatedEnvironmentIds).toEqual(["env_prod"]); + }); +}); diff --git a/apps/webapp/test/member.server.test.ts b/apps/webapp/test/member.server.test.ts index 722c49aff18..dd82f1e98a5 100644 --- a/apps/webapp/test/member.server.test.ts +++ b/apps/webapp/test/member.server.test.ts @@ -520,8 +520,8 @@ describe("provisionMemberDevelopmentEnvironments", () => { }); await provisionMemberDevelopmentEnvironments({ + source: "invite", inviteId: invite.id, - user: { id: invitee.id, email: invitee.email }, member, organization, projects: activeProjects, @@ -545,7 +545,7 @@ describe("provisionMemberDevelopmentEnvironments", () => { { timeout: 60_000 }, async ({ prisma }) => { prismaHolder.client = prisma; - const { provisionMemberDevelopmentEnvironments, ENV_SETUP_INCOMPLETE } = + const { provisionMemberDevelopmentEnvironments, DevEnvironmentProvisioningError } = await import("../app/models/member.server"); const { invitee, organization, activeProjects, invite } = await seedInviteFixture(prisma, { @@ -564,14 +564,13 @@ describe("provisionMemberDevelopmentEnvironments", () => { await expect( provisionMemberDevelopmentEnvironments({ - inviteId: invite.id, - user: { id: invitee.id, email: invitee.email }, + source: "sso_jit", member, organization, projects: [...activeProjects, { id: "missing-project-id" }], maximumConcurrencyLimit: 5, }) - ).rejects.toThrow(ENV_SETUP_INCOMPLETE); + ).rejects.toThrow(DevEnvironmentProvisioningError); const devEnvs = await prisma.runtimeEnvironment.findMany({ where: { @@ -588,6 +587,57 @@ describe("provisionMemberDevelopmentEnvironments", () => { ); }); +describe("acceptInvite environment provisioning failures", () => { + postgresTest( + "reports setup as incomplete and keeps the invite so it can be retried", + { timeout: 60_000 }, + async ({ prisma }) => { + prismaHolder.client = prisma; + const { acceptInvite, ENV_SETUP_INCOMPLETE, isAcceptInviteFormError } = + await import("../app/models/member.server"); + + const { invitee, organization, activeProjects, invite } = await seedInviteFixture(prisma, { + activeProjectCount: 2, + }); + + const member = await prisma.orgMember.create({ + data: { + organizationId: organization.id, + userId: invitee.id, + role: "MEMBER", + }, + }); + + const keys = devEnvKeys(`tr_stg_${randomHex(24)}`, `pk_stg_${randomHex(24)}`); + await prisma.runtimeEnvironment.create({ + data: { + slug: "dev", + type: "STAGING", + ...keys, + projectId: activeProjects[1].id, + organizationId: organization.id, + orgMemberId: member.id, + }, + }); + + const error = await acceptInvite({ + inviteId: invite.id, + organizationId: organization.id, + user: { id: invitee.id, email: invitee.email }, + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toBe(ENV_SETUP_INCOMPLETE); + expect(isAcceptInviteFormError(error)).toBe(true); + + const remainingInvite = await prisma.orgMemberInvite.findFirst({ + where: { id: invite.id }, + }); + expect(remainingInvite).not.toBeNull(); + } + ); +}); + describe("acceptInvite recovery", () => { postgresTest( "retries successfully when membership exists and the invite is still pending", diff --git a/apps/webapp/test/memberDevEnvironments.server.test.ts b/apps/webapp/test/memberDevEnvironments.server.test.ts new file mode 100644 index 00000000000..57d7b07872c --- /dev/null +++ b/apps/webapp/test/memberDevEnvironments.server.test.ts @@ -0,0 +1,411 @@ +import { randomBytes } from "node:crypto"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { PrismaClient } from "@trigger.dev/database"; + +const prismaHolder = vi.hoisted(() => ({ + client: null as PrismaClient | null, +})); + +vi.mock("~/services/rbac.server", () => ({ + rbac: { + getUserRole: async () => null, + setUserRole: async () => ({ ok: true }), + }, +})); + +const workerHolder = vi.hoisted(() => ({ + calls: [] as Array<{ id: string; job: string; payload: unknown }>, + shouldThrow: false, +})); +vi.mock("~/v3/commonWorker.server", () => ({ + commonWorker: { + enqueueOnce: async (args: { id: string; job: string; payload: unknown }) => { + if (workerHolder.shouldThrow) { + throw new Error("redis unavailable"); + } + workerHolder.calls.push(args); + }, + }, +})); + +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + + return { + Prisma, + get prisma() { + if (!prismaHolder.client) { + throw new Error("test prisma not set"); + } + return prismaHolder.client; + }, + get $replica() { + if (!prismaHolder.client) { + throw new Error("test prisma not set"); + } + return prismaHolder.client; + }, + }; +}); + +import { postgresTest } from "@internal/testcontainers"; + +vi.setConfig({ testTimeout: 60_000 }); + +function randomHex(len = 12): string { + return randomBytes(Math.ceil(len / 2)) + .toString("hex") + .slice(0, len); +} + +async function seedMembershipFixture( + prisma: PrismaClient, + opts: { activeProjectCount: number; deletedProjectCount?: number } +) { + const suffix = randomHex(8); + + const user = await prisma.user.create({ + data: { + email: `member-${suffix}@test.local`, + authenticationMethod: "SSO", + }, + }); + + const organization = await prisma.organization.create({ + data: { + title: `sso-org-${suffix}`, + slug: `sso-org-${suffix}`, + isActivated: true, + members: { create: { userId: user.id, role: "MEMBER" } }, + }, + }); + + const activeProjects = []; + for (let i = 0; i < opts.activeProjectCount; i++) { + activeProjects.push( + await prisma.project.create({ + data: { + name: `active-project-${i}-${suffix}`, + slug: `active-proj-${i}-${suffix}`, + externalRef: `proj_active_${i}_${suffix}`, + organizationId: organization.id, + engine: "V2", + }, + }) + ); + } + + for (let i = 0; i < (opts.deletedProjectCount ?? 0); i++) { + await prisma.project.create({ + data: { + name: `deleted-project-${i}-${suffix}`, + slug: `deleted-proj-${i}-${suffix}`, + externalRef: `proj_deleted_${i}_${suffix}`, + organizationId: organization.id, + engine: "V2", + deletedAt: new Date(), + }, + }); + } + + return { user, organization, activeProjects }; +} + +function devEnvironmentsFor(prisma: PrismaClient, organizationId: string) { + return prisma.runtimeEnvironment.findMany({ + where: { organizationId, type: "DEVELOPMENT" }, + select: { projectId: true, orgMemberId: true, slug: true }, + }); +} + +describe("provisionDevEnvironmentsForMembership", () => { + postgresTest("creates a development environment for every active project", async ({ prisma }) => { + prismaHolder.client = prisma; + const { provisionDevEnvironmentsForMembership } = + await import("../app/services/memberDevEnvironments.server"); + + const { user, organization, activeProjects } = await seedMembershipFixture(prisma, { + activeProjectCount: 3, + }); + + await provisionDevEnvironmentsForMembership({ + userId: user.id, + organizationId: organization.id, + source: "sso_jit", + }); + + const devEnvs = await devEnvironmentsFor(prisma, organization.id); + const member = await prisma.orgMember.findFirstOrThrow({ + where: { userId: user.id, organizationId: organization.id }, + }); + + expect(devEnvs.map((env) => env.projectId).sort()).toEqual( + activeProjects.map((project) => project.id).sort() + ); + expect(devEnvs.every((env) => env.orgMemberId === member.id)).toBe(true); + expect(devEnvs.every((env) => env.slug === "dev")).toBe(true); + }); + + postgresTest("skips soft-deleted projects", async ({ prisma }) => { + prismaHolder.client = prisma; + const { provisionDevEnvironmentsForMembership } = + await import("../app/services/memberDevEnvironments.server"); + + const { user, organization, activeProjects } = await seedMembershipFixture(prisma, { + activeProjectCount: 2, + deletedProjectCount: 2, + }); + + await provisionDevEnvironmentsForMembership({ + userId: user.id, + organizationId: organization.id, + source: "directory_sync", + }); + + const devEnvs = await devEnvironmentsFor(prisma, organization.id); + + expect(devEnvs.map((env) => env.projectId).sort()).toEqual( + activeProjects.map((project) => project.id).sort() + ); + }); + + postgresTest("creates no duplicates when it runs again", async ({ prisma }) => { + prismaHolder.client = prisma; + const { provisionDevEnvironmentsForMembership } = + await import("../app/services/memberDevEnvironments.server"); + + const { user, organization, activeProjects } = await seedMembershipFixture(prisma, { + activeProjectCount: 2, + }); + + const payload = { + userId: user.id, + organizationId: organization.id, + source: "sso_jit" as const, + }; + + await provisionDevEnvironmentsForMembership(payload); + await provisionDevEnvironmentsForMembership(payload); + + const devEnvs = await devEnvironmentsFor(prisma, organization.id); + + expect(devEnvs).toHaveLength(activeProjects.length); + }); + + postgresTest("creates only the environments that are missing", async ({ prisma }) => { + prismaHolder.client = prisma; + const { provisionDevEnvironmentsForMembership } = + await import("../app/services/memberDevEnvironments.server"); + + const { user, organization, activeProjects } = await seedMembershipFixture(prisma, { + activeProjectCount: 3, + }); + const member = await prisma.orgMember.findFirstOrThrow({ + where: { userId: user.id, organizationId: organization.id }, + }); + + const existing = await prisma.runtimeEnvironment.create({ + data: { + slug: "dev", + type: "DEVELOPMENT", + apiKey: `tr_dev_${randomHex(24)}`, + pkApiKey: `pk_dev_${randomHex(24)}`, + shortcode: randomHex(4), + projectId: activeProjects[1].id, + organizationId: organization.id, + orgMemberId: member.id, + }, + }); + + await provisionDevEnvironmentsForMembership({ + userId: user.id, + organizationId: organization.id, + source: "sso_jit", + }); + + const devEnvs = await prisma.runtimeEnvironment.findMany({ + where: { organizationId: organization.id, type: "DEVELOPMENT" }, + select: { id: true, projectId: true }, + }); + + expect(devEnvs).toHaveLength(activeProjects.length); + expect(devEnvs.find((env) => env.projectId === activeProjects[1].id)?.id).toBe(existing.id); + }); + + postgresTest("does nothing when the membership no longer exists", async ({ prisma }) => { + prismaHolder.client = prisma; + const { provisionDevEnvironmentsForMembership } = + await import("../app/services/memberDevEnvironments.server"); + + const { user, organization } = await seedMembershipFixture(prisma, { + activeProjectCount: 2, + }); + + await prisma.orgMember.deleteMany({ + where: { userId: user.id, organizationId: organization.id }, + }); + + await expect( + provisionDevEnvironmentsForMembership({ + userId: user.id, + organizationId: organization.id, + source: "directory_sync", + }) + ).resolves.toBeUndefined(); + + expect(await devEnvironmentsFor(prisma, organization.id)).toHaveLength(0); + }); +}); + +describe("createDevelopmentEnvironmentForMember", () => { + postgresTest("creates the environment when the member has none", async ({ prisma }) => { + prismaHolder.client = prisma; + const { createDevelopmentEnvironmentForMember } = + await import("../app/models/organization.server"); + + const { user, organization, activeProjects } = await seedMembershipFixture(prisma, { + activeProjectCount: 1, + }); + const member = await prisma.orgMember.findFirstOrThrow({ + where: { userId: user.id, organizationId: organization.id }, + }); + + const result = await createDevelopmentEnvironmentForMember({ + organization, + project: activeProjects[0], + member, + maximumConcurrencyLimit: 5, + }); + + expect(result.created).toBe(true); + expect(await devEnvironmentsFor(prisma, organization.id)).toHaveLength(1); + }); + + postgresTest( + "treats a concurrently created environment as already provisioned", + async ({ prisma }) => { + prismaHolder.client = prisma; + const { createDevelopmentEnvironmentForMember } = + await import("../app/models/organization.server"); + + const { user, organization, activeProjects } = await seedMembershipFixture(prisma, { + activeProjectCount: 1, + }); + const member = await prisma.orgMember.findFirstOrThrow({ + where: { userId: user.id, organizationId: organization.id }, + }); + + const args = { + organization, + project: activeProjects[0], + member, + maximumConcurrencyLimit: 5, + }; + + const results = await Promise.all([ + createDevelopmentEnvironmentForMember(args), + createDevelopmentEnvironmentForMember(args), + ]); + + expect(results.filter((result) => result.created)).toHaveLength(1); + expect(results.filter((result) => !result.created)).toHaveLength(1); + expect(await devEnvironmentsFor(prisma, organization.id)).toHaveLength(1); + } + ); + + postgresTest( + "rethrows when the conflict is not a development environment", + async ({ prisma }) => { + prismaHolder.client = prisma; + const { createDevelopmentEnvironmentForMember } = + await import("../app/models/organization.server"); + + const { user, organization, activeProjects } = await seedMembershipFixture(prisma, { + activeProjectCount: 1, + }); + const member = await prisma.orgMember.findFirstOrThrow({ + where: { userId: user.id, organizationId: organization.id }, + }); + + await prisma.runtimeEnvironment.create({ + data: { + slug: "dev", + type: "STAGING", + apiKey: `tr_stg_${randomHex(24)}`, + pkApiKey: `pk_stg_${randomHex(24)}`, + shortcode: randomHex(4), + projectId: activeProjects[0].id, + organizationId: organization.id, + orgMemberId: member.id, + }, + }); + + await expect( + createDevelopmentEnvironmentForMember({ + organization, + project: activeProjects[0], + member, + maximumConcurrencyLimit: 5, + }) + ).rejects.toThrow(); + } + ); +}); + +describe("enqueueMemberDevelopmentEnvironments", () => { + beforeEach(() => { + workerHolder.calls.length = 0; + workerHolder.shouldThrow = false; + }); + + it("dedupes by organization and user so repeat sign-ins collapse", async () => { + const { enqueueMemberDevelopmentEnvironments } = + await import("../app/services/memberDevEnvironments.server"); + + const result = await enqueueMemberDevelopmentEnvironments({ + userId: "user_1", + organizationId: "org_1", + source: "sso_jit", + }); + + expect(result).toEqual({ enqueued: true }); + expect(workerHolder.calls).toEqual([ + { + id: "membership:devEnvs:org_1:user_1", + job: "membership.provisionDevEnvironments", + payload: { userId: "user_1", organizationId: "org_1", source: "sso_jit" }, + }, + ]); + }); + + it("reports a queue failure instead of throwing", async () => { + const { enqueueMemberDevelopmentEnvironments } = + await import("../app/services/memberDevEnvironments.server"); + workerHolder.shouldThrow = true; + + const result = await enqueueMemberDevelopmentEnvironments({ + userId: "user_1", + organizationId: "org_1", + source: "directory_sync", + }); + + expect(result).toEqual({ enqueued: false }); + }); +}); + +describe("membership source schema", () => { + it("accepts every membership source the job can be enqueued with", async () => { + const { MembershipSourceSchema } = await import("../app/models/member.server"); + const { MembershipDevEnvironmentsSchema } = + await import("../app/services/memberDevEnvironments.server"); + + for (const source of MembershipSourceSchema.options) { + const parsed = MembershipDevEnvironmentsSchema.safeParse({ + userId: "user_1", + organizationId: "org_1", + source, + }); + expect(parsed.success, `source "${source}" must be accepted by the job schema`).toBe(true); + } + }); +}); diff --git a/apps/webapp/test/orgMember.server.test.ts b/apps/webapp/test/orgMember.server.test.ts new file mode 100644 index 00000000000..e5664c9497a --- /dev/null +++ b/apps/webapp/test/orgMember.server.test.ts @@ -0,0 +1,182 @@ +import { randomBytes } from "node:crypto"; +import { beforeEach, describe, expect, vi } from "vitest"; +import type { PrismaClient } from "@trigger.dev/database"; + +const prismaHolder = vi.hoisted(() => ({ + client: null as PrismaClient | null, +})); + +type SetUserRoleResult = { ok: true } | { ok: false; error: string; code?: "last_owner" }; + +const rbacHolder = vi.hoisted(() => ({ + setUserRoleResult: { ok: true } as SetUserRoleResult, + currentRole: null as { id: string } | null, +})); + +vi.mock("~/services/rbac.server", () => ({ + rbac: { + getUserRole: async () => rbacHolder.currentRole, + setUserRole: async () => rbacHolder.setUserRoleResult, + }, +})); + +const enqueueHolder = vi.hoisted(() => ({ + calls: [] as unknown[], + enqueued: true, +})); +vi.mock("~/services/memberDevEnvironments.server", () => ({ + enqueueMemberDevelopmentEnvironments: async (payload: unknown) => { + enqueueHolder.calls.push(payload); + return { enqueued: enqueueHolder.enqueued }; + }, +})); +const enqueueCalls = enqueueHolder.calls; + +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + + return { + Prisma, + get prisma() { + if (!prismaHolder.client) { + throw new Error("test prisma not set"); + } + return prismaHolder.client; + }, + get $replica() { + if (!prismaHolder.client) { + throw new Error("test prisma not set"); + } + return prismaHolder.client; + }, + }; +}); + +import { postgresTest } from "@internal/testcontainers"; + +vi.setConfig({ testTimeout: 60_000 }); + +beforeEach(() => { + enqueueCalls.length = 0; + enqueueHolder.enqueued = true; + rbacHolder.setUserRoleResult = { ok: true }; + rbacHolder.currentRole = null; +}); + +function randomHex(len = 12): string { + return randomBytes(Math.ceil(len / 2)) + .toString("hex") + .slice(0, len); +} + +async function seedUserAndOrg(prisma: PrismaClient) { + const suffix = randomHex(8); + + const owner = await prisma.user.create({ + data: { email: `owner-${suffix}@test.local`, authenticationMethod: "MAGIC_LINK" }, + }); + const user = await prisma.user.create({ + data: { email: `joiner-${suffix}@test.local`, authenticationMethod: "SSO" }, + }); + const organization = await prisma.organization.create({ + data: { + title: `jit-org-${suffix}`, + slug: `jit-org-${suffix}`, + isActivated: true, + members: { create: { userId: owner.id, role: "ADMIN" } }, + }, + }); + + return { user, organization }; +} + +describe("ensureOrgMember development environment provisioning", () => { + postgresTest("queues provisioning for a newly created membership", async ({ prisma }) => { + prismaHolder.client = prisma; + const { ensureOrgMember } = await import("../app/models/orgMember.server"); + + const { user, organization } = await seedUserAndOrg(prisma); + + const result = await ensureOrgMember({ + userId: user.id, + organizationId: organization.id, + roleId: null, + source: "sso_jit", + }); + + expect(result.created).toBe(true); + expect(enqueueCalls).toEqual([ + { userId: user.id, organizationId: organization.id, source: "sso_jit" }, + ]); + }); + + postgresTest("queues provisioning for a membership that already exists", async ({ prisma }) => { + prismaHolder.client = prisma; + const { ensureOrgMember } = await import("../app/models/orgMember.server"); + + const { user, organization } = await seedUserAndOrg(prisma); + await prisma.orgMember.create({ + data: { userId: user.id, organizationId: organization.id, role: "MEMBER" }, + }); + + const result = await ensureOrgMember({ + userId: user.id, + organizationId: organization.id, + roleId: null, + source: "directory_sync", + }); + + expect(result.created).toBe(false); + expect(enqueueCalls).toEqual([ + { userId: user.id, organizationId: organization.id, source: "directory_sync" }, + ]); + }); + + postgresTest( + "does not queue provisioning when the membership is rolled back", + async ({ prisma }) => { + prismaHolder.client = prisma; + const { ensureOrgMember } = await import("../app/models/orgMember.server"); + + const { user, organization } = await seedUserAndOrg(prisma); + rbacHolder.setUserRoleResult = { ok: false, error: "role service unavailable" }; + + await expect( + ensureOrgMember({ + userId: user.id, + organizationId: organization.id, + roleId: "role_restricted", + source: "sso_jit", + }) + ).rejects.toThrow(/failed to apply role/); + + const member = await prisma.orgMember.findFirst({ + where: { userId: user.id, organizationId: organization.id }, + }); + expect(member).toBeNull(); + expect(enqueueCalls).toEqual([]); + } + ); + + postgresTest("reports a failed enqueue without failing the membership", async ({ prisma }) => { + prismaHolder.client = prisma; + const { ensureOrgMember } = await import("../app/models/orgMember.server"); + + const { user, organization } = await seedUserAndOrg(prisma); + enqueueHolder.enqueued = false; + + const result = await ensureOrgMember({ + userId: user.id, + organizationId: organization.id, + roleId: null, + source: "directory_sync", + }); + + expect(result).toMatchObject({ created: true, devEnvironmentsQueued: false }); + + const member = await prisma.orgMember.findFirst({ + where: { userId: user.id, organizationId: organization.id }, + }); + expect(member).not.toBeNull(); + }); +}); diff --git a/apps/webapp/test/selectBestEnvironment.test.ts b/apps/webapp/test/selectBestEnvironment.test.ts new file mode 100644 index 00000000000..2174a17af92 --- /dev/null +++ b/apps/webapp/test/selectBestEnvironment.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it, vi } from "vitest"; +import type { UserFromSession } from "~/services/session.server"; + +vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} })); + +import { SelectBestEnvironmentPresenter } from "~/presenters/SelectBestEnvironmentPresenter.server"; + +const PROJECT_ID = "project_1"; +const USER_ID = "user_me"; +const OTHER_USER_ID = "user_colleague"; + +function userWithPreference(environmentId: string | undefined): UserFromSession { + return { + id: USER_ID, + dashboardPreferences: { + currentProjectId: PROJECT_ID, + projects: environmentId + ? { [PROJECT_ID]: { currentEnvironment: { id: environmentId } } } + : {}, + }, + } as unknown as UserFromSession; +} + +const myDev = { + id: "env_my_dev", + type: "DEVELOPMENT" as const, + slug: "dev", + parentEnvironmentId: null, + orgMember: { userId: USER_ID }, +}; + +const theirDev = { + id: "env_their_dev", + type: "DEVELOPMENT" as const, + slug: "dev", + parentEnvironmentId: null, + orgMember: { userId: OTHER_USER_ID }, +}; + +const prod = { + id: "env_prod", + type: "PRODUCTION" as const, + slug: "prod", + parentEnvironmentId: null, + orgMember: null, +}; + +describe("SelectBestEnvironmentPresenter.selectBestEnvironment", () => { + const presenter = new SelectBestEnvironmentPresenter({} as never); + + it("honours a stored preference that belongs to the user", async () => { + const result = await presenter.selectBestEnvironment(PROJECT_ID, userWithPreference(myDev.id), [ + theirDev, + myDev, + prod, + ]); + + expect(result).toBe(myDev); + }); + + it("ignores a stored preference pointing at another member's dev environment", async () => { + const result = await presenter.selectBestEnvironment( + PROJECT_ID, + userWithPreference(theirDev.id), + [theirDev, myDev, prod] + ); + + expect(result).toBe(myDev); + }); + + it("falls back to production when the user has no dev environment of their own", async () => { + const result = await presenter.selectBestEnvironment( + PROJECT_ID, + userWithPreference(theirDev.id), + [theirDev, prod] + ); + + expect(result).toBe(prod); + }); + + it("returns the user's own dev environment when no preference is stored", async () => { + const result = await presenter.selectBestEnvironment( + PROJECT_ID, + userWithPreference(undefined), + [theirDev, myDev, prod] + ); + + expect(result).toBe(myDev); + }); +}); From 68ed809416f3dea8a869ec6a8b9764c6c8150326 Mon Sep 17 00:00:00 2001 From: Chris Arderne <chris@trigger.dev> Date: Thu, 30 Jul 2026 21:20:44 +0100 Subject: [PATCH 42/67] test(clickhouse): keep queue metrics fixtures within TTL (#4428) ## Summary Keeps the queue metrics ClickHouse tests stable as wall-clock time advances. ## Root cause The fixtures used fixed timestamps. Once those timestamps crossed the tables' 30-day retention boundary, ClickHouse immediately expired the inserted aggregate rows and all six tests read empty results. The fixtures now derive a recent minute-aligned timestamp once per test file. The second 10-second bucket and ranking window are derived from the same anchor, preserving deterministic bucket relationships while keeping rows inside both the raw and aggregate table TTLs. Verified with `pnpm --filter @internal/clickhouse exec vitest run src/queueMetrics.test.ts`. --- .../clickhouse/src/queueMetrics.test.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/internal-packages/clickhouse/src/queueMetrics.test.ts b/internal-packages/clickhouse/src/queueMetrics.test.ts index 00532041e44..e8b138a8529 100644 --- a/internal-packages/clickhouse/src/queueMetrics.test.ts +++ b/internal-packages/clickhouse/src/queueMetrics.test.ts @@ -6,7 +6,17 @@ import type { QueueMetricsRawV1Input } from "./queueMetrics.js"; const ORG = "org_qm"; const PROJECT = "project_qm"; const ENV = "env_qm"; -const EVENT_TIME = "2026-06-30 12:00:05"; // all rows land in the 10s bucket starting 12:00:00 + +function clickhouseDateTime(timestamp: number) { + return new Date(timestamp).toISOString().slice(0, 19).replace("T", " "); +} + +// Keep fixtures within the tables' TTLs while preserving deterministic bucket +// relationships. Subtract a minute before rounding so no fixture lands in the future. +const TEST_MINUTE = Math.floor((Date.now() - 60_000) / 60_000) * 60_000; +const EVENT_TIME = clickhouseDateTime(TEST_MINUTE + 5_000); +const NEXT_BUCKET_EVENT_TIME = clickhouseDateTime(TEST_MINUTE + 15_000); +const RANKING_START_TIME = clickhouseDateTime(TEST_MINUTE - 10 * 60_000); function base(op: QueueMetricsRawV1Input["op"], queue: string): QueueMetricsRawV1Input { return { @@ -246,7 +256,7 @@ describe("queue_metrics_v1", () => { { ...base("gauge", "roll-b"), running: 2, queued: 1, env_running: 45, env_limit: 50 }, { ...base("gauge", "roll-a"), - event_time: "2026-06-30 12:00:15", + event_time: NEXT_BUCKET_EVENT_TIME, running: 1, queued: 2, env_running: 20, @@ -352,7 +362,7 @@ describe("queue_metrics_v1", () => { organizationId: ORG, projectId: PROJECT, environmentId: ENV, - startTime: "2026-06-30 11:50:00", + startTime: RANKING_START_TIME, nameContains: "rank-", byQueuedOnly: 0, }; From debfa2b733243700feb2fd9fe34acb730cb1dcc4 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <209825114+claude[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 21:28:44 +0100 Subject: [PATCH 43/67] feat(webapp): impersonation consent page and a view-as-user toggle (#4421) --- .../impersonation-consent-and-view-as-user.md | 6 + .../app/components/ImpersonationBanner.tsx | 30 --- apps/webapp/app/components/admin/debugRun.tsx | 6 +- .../app/components/admin/debugTooltip.tsx | 7 +- .../app/components/navigation/SideMenu.tsx | 89 +++++++-- apps/webapp/app/hooks/useUser.ts | 16 +- apps/webapp/app/models/admin.server.ts | 89 ++++++++- apps/webapp/app/root.tsx | 12 ++ .../routes/_app.@.orgs.$organizationSlug.$.ts | 82 -------- .../_app.@.orgs.$organizationSlug.$.tsx | 188 ++++++++++++++++++ .../route.tsx | 4 +- .../route.tsx | 3 + .../route.tsx | 11 +- .../route.tsx | 7 +- .../route.tsx | 3 + .../route.tsx | 9 +- .../resources.impersonation_.view-as.ts | 50 +++++ ...ectParam.env.$envParam.runs.bulkaction.tsx | 8 +- .../resources.taskruns.$runParam.replay.ts | 5 + .../app/services/impersonation.server.ts | 53 ++++- apps/webapp/app/services/session.server.ts | 37 +++- .../app/utils/impersonationPaths.test.ts | 119 +++++++++++ .../app/utils/impersonationState.test.ts | 71 +++++++ apps/webapp/app/utils/impersonationState.ts | 47 +++++ apps/webapp/app/utils/pathBuilder.ts | 35 ++++ apps/webapp/test/impersonationConsent.test.ts | 153 ++++++++++++++ .../runGetRoutes.replicaLag.guard.test.ts | 12 +- apps/webapp/test/viewAsUser.test.ts | 109 ++++++++++ 28 files changed, 1096 insertions(+), 165 deletions(-) create mode 100644 .server-changes/impersonation-consent-and-view-as-user.md delete mode 100644 apps/webapp/app/components/ImpersonationBanner.tsx delete mode 100644 apps/webapp/app/routes/_app.@.orgs.$organizationSlug.$.ts create mode 100644 apps/webapp/app/routes/_app.@.orgs.$organizationSlug.$.tsx create mode 100644 apps/webapp/app/routes/resources.impersonation_.view-as.ts create mode 100644 apps/webapp/app/utils/impersonationPaths.test.ts create mode 100644 apps/webapp/app/utils/impersonationState.test.ts create mode 100644 apps/webapp/app/utils/impersonationState.ts create mode 100644 apps/webapp/test/impersonationConsent.test.ts create mode 100644 apps/webapp/test/viewAsUser.test.ts diff --git a/.server-changes/impersonation-consent-and-view-as-user.md b/.server-changes/impersonation-consent-and-view-as-user.md new file mode 100644 index 00000000000..232e97e8b90 --- /dev/null +++ b/.server-changes/impersonation-consent-and-view-as-user.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Admins opening an impersonation link from outside the dashboard now get a confirmation page naming the organization and destination instead of being bounced back, and while impersonating they can switch to "View as user" to see the dashboard exactly as that user sees it, with the admin-only UI and the impersonation highlight both hidden. Stopping impersonation is still one click away in the account menu. diff --git a/apps/webapp/app/components/ImpersonationBanner.tsx b/apps/webapp/app/components/ImpersonationBanner.tsx deleted file mode 100644 index c16e822df1e..00000000000 --- a/apps/webapp/app/components/ImpersonationBanner.tsx +++ /dev/null @@ -1,30 +0,0 @@ -import { Form } from "@remix-run/react"; -import { UserCrossIcon } from "~/assets/icons/UserCrossIcon"; -import { Button } from "./primitives/Buttons"; -import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "./primitives/Tooltip"; - -export function ImpersonationBanner() { - return ( - <div> - <Form action="/resources/impersonation" method="delete" reloadDocument> - <TooltipProvider disableHoverableContent={true}> - <Tooltip> - <TooltipTrigger asChild> - <Button - type="submit" - variant="small-menu-item" - LeadingIcon={UserCrossIcon} - fullWidth - textAlignLeft - className="text-amber-400" - /> - </TooltipTrigger> - <TooltipContent side="bottom" className={"text-xs"}> - Stop impersonating - </TooltipContent> - </Tooltip> - </TooltipProvider> - </Form> - </div> - ); -} diff --git a/apps/webapp/app/components/admin/debugRun.tsx b/apps/webapp/app/components/admin/debugRun.tsx index 705e3169a8b..049c5cd08c3 100644 --- a/apps/webapp/app/components/admin/debugRun.tsx +++ b/apps/webapp/app/components/admin/debugRun.tsx @@ -1,4 +1,3 @@ -import { useIsImpersonating } from "~/hooks/useOrganizations"; import { useHasAdminAccess } from "~/hooks/useUser"; import { Button } from "../primitives/Buttons"; import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "../primitives/Dialog"; @@ -12,10 +11,11 @@ import * as Property from "~/components/primitives/PropertyTable"; import { ClipboardField } from "../primitives/ClipboardField"; export function AdminDebugRun({ friendlyId }: { friendlyId: string }) { + // `useHasAdminAccess` already folds in impersonation and the "view as user" + // toggle, so this one check is enough. const hasAdminAccess = useHasAdminAccess(); - const isImpersonating = useIsImpersonating(); - if (!hasAdminAccess && !isImpersonating) { + if (!hasAdminAccess) { return null; } diff --git a/apps/webapp/app/components/admin/debugTooltip.tsx b/apps/webapp/app/components/admin/debugTooltip.tsx index 4157f898163..1729bfa740c 100644 --- a/apps/webapp/app/components/admin/debugTooltip.tsx +++ b/apps/webapp/app/components/admin/debugTooltip.tsx @@ -8,15 +8,16 @@ import { TooltipTrigger, } from "~/components/primitives/Tooltip"; import { useOptionalEnvironment } from "~/hooks/useEnvironment"; -import { useIsImpersonating, useOptionalOrganization } from "~/hooks/useOrganizations"; +import { useOptionalOrganization } from "~/hooks/useOrganizations"; import { useOptionalProject } from "~/hooks/useProject"; import { useHasAdminAccess, useUser } from "~/hooks/useUser"; export function AdminDebugTooltip({ children }: { children?: React.ReactNode }) { + // `useHasAdminAccess` already folds in impersonation and the "view as user" + // toggle, so this one check is enough. const hasAdminAccess = useHasAdminAccess(); - const isImpersonating = useIsImpersonating(); - if (!hasAdminAccess && !isImpersonating) { + if (!hasAdminAccess) { return null; } diff --git a/apps/webapp/app/components/navigation/SideMenu.tsx b/apps/webapp/app/components/navigation/SideMenu.tsx index 29eb1fc8520..ebedc77fc6e 100644 --- a/apps/webapp/app/components/navigation/SideMenu.tsx +++ b/apps/webapp/app/components/navigation/SideMenu.tsx @@ -4,7 +4,14 @@ import { ExclamationTriangleIcon, } from "@heroicons/react/24/outline"; import { EllipsisHorizontalIcon } from "@heroicons/react/20/solid"; -import { useFetcher, useNavigation, useRevalidator, useSubmit } from "@remix-run/react"; +import { + Form, + useFetcher, + useLocation, + useNavigation, + useRevalidator, + useSubmit, +} from "@remix-run/react"; import { LayoutGroup, motion } from "framer-motion"; import { type CSSProperties, @@ -33,6 +40,8 @@ import { DeploymentsIcon } from "~/assets/icons/DeploymentsIcon"; import { DialIcon } from "~/assets/icons/DialIcon"; import { DropdownIcon } from "~/assets/icons/DropdownIcon"; import { BranchEnvironmentIconSmall } from "~/assets/icons/EnvironmentIcons"; +import { EyeClosedIcon } from "~/assets/icons/EyeClosedIcon"; +import { EyeOpenIcon } from "~/assets/icons/EyeOpenIcon"; import { FolderClosedIcon } from "~/assets/icons/FolderClosedIcon"; import { FolderOpenIcon } from "~/assets/icons/FolderOpenIcon"; import { GlobeLinesIcon } from "~/assets/icons/GlobeLinesIcon"; @@ -69,7 +78,7 @@ import { type MatchedOrganization } from "~/hooks/useOrganizations"; import { type MatchedProject } from "~/hooks/useProject"; import { useShortcutKeys } from "~/hooks/useShortcutKeys"; import { useShowSelfServe } from "~/hooks/useShowSelfServe"; -import { useHasAdminAccess } from "~/hooks/useUser"; +import { useHasAdminAccess, useIsViewingAsUser } from "~/hooks/useUser"; import { type UserWithDashboardPreferences } from "~/models/user.server"; import { useCurrentPlan, @@ -389,6 +398,7 @@ export function SideMenu({ const { isConnected } = useDevPresence(); const isFreeUser = currentPlan?.v3Subscription?.isPaying === false; const isAdmin = useHasAdminAccess(); + const isViewingAsUser = useIsViewingAsUser(); const { isManagedCloud } = useFeatures(); const featureFlags = useFeatureFlags(); const incidentStatus = useIncidentStatus(); @@ -785,7 +795,7 @@ export function SideMenu({ // user's saved order/hidden preferences are applied at render below. const staticSections: SideMenuSectionConfig[] = []; - if (user.admin || user.isImpersonating || featureFlags.hasAiAccess) { + if (isAdmin || featureFlags.hasAiAccess) { staticSections.push({ id: "ai", title: "AI", @@ -813,12 +823,12 @@ export function SideMenu({ }); } - if (user.admin || user.isImpersonating || featureFlags.hasQueryAccess) { + if (isAdmin || featureFlags.hasQueryAccess) { staticSections.push({ id: "metrics", title: "Observability", items: [ - ...(user.admin || user.isImpersonating || featureFlags.hasLogsPageAccess + ...(isAdmin || featureFlags.hasLogsPageAccess ? [ { id: "logs", @@ -1048,7 +1058,13 @@ export function SideMenu({ style={initialStyleRef.current} className={cn( "relative h-full border-r bg-background-bright", - user.isImpersonating ? IMPERSONATION_ACCENT.border : "border-grid-bright" + // The accent is the loudest "you are not this user" tell, so "view as user" drops it too — + // the point of the mode is a dashboard that looks exactly like the user's. The account + // menu's "Stop impersonating" and the toggle itself stay on raw impersonation, so there is + // still a way back out (as does the ⌘⌥A shortcut in <GlobalShortcuts>). + user.isImpersonating && !isViewingAsUser + ? IMPERSONATION_ACCENT.border + : "border-grid-bright" )} > <ResizeHandle @@ -1832,24 +1848,29 @@ function AccountMenuItems({ return ( <> - {isAdmin && ( + {/* "Stop impersonating" and the view-as-user toggle key off raw impersonation, not `isAdmin`: + with "view as user" on, `isAdmin` is false and these are the only ways back out. */} + {(isImpersonating || isAdmin) && ( <div className="flex flex-col gap-1 border-b border-grid-bright p-1"> {isImpersonating ? ( - <PopoverMenuItem - title={ - <div className="flex w-full items-center justify-between"> - <span className={IMPERSONATION_ACCENT.text}>Stop impersonating</span> - <ShortcutKey - shortcut={{ modifiers: ["mod", "alt"], key: "a" }} - variant="medium/bright" - /> - </div> - } - icon={UserCrossIcon} - onClick={stopImpersonating} - leadingIconClassName={cn(SIDE_MENU_POPOVER_ITEM_ICON, IMPERSONATION_ACCENT.text)} - className={SIDE_MENU_POPOVER_ITEM_LABEL} - /> + <> + <PopoverMenuItem + title={ + <div className="flex w-full items-center justify-between"> + <span className={IMPERSONATION_ACCENT.text}>Stop impersonating</span> + <ShortcutKey + shortcut={{ modifiers: ["mod", "alt"], key: "a" }} + variant="medium/bright" + /> + </div> + } + icon={UserCrossIcon} + onClick={stopImpersonating} + leadingIconClassName={cn(SIDE_MENU_POPOVER_ITEM_ICON, IMPERSONATION_ACCENT.text)} + className={SIDE_MENU_POPOVER_ITEM_LABEL} + /> + <ViewAsUserMenuItem /> + </> ) : ( <PopoverMenuItem to={adminPath()} @@ -1906,6 +1927,30 @@ function AccountMenuItems({ ); } +/** + * Toggles the display-only "view as user" mode for the current impersonation session, so an admin + * can see the dashboard the way the impersonated user sees it. `reloadDocument` forces a full + * navigation, so every loader re-runs under the updated cookie instead of reusing cached data. + */ +function ViewAsUserMenuItem() { + const isViewingAsUser = useIsViewingAsUser(); + const location = useLocation(); + + return ( + <Form method="post" action="/resources/impersonation/view-as" reloadDocument> + <input type="hidden" name="viewAsUser" value={isViewingAsUser ? "false" : "true"} /> + <input type="hidden" name="redirectTo" value={`${location.pathname}${location.search}`} /> + <PopoverMenuItem + type="submit" + title={isViewingAsUser ? "Show admin UI" : "View as user"} + icon={isViewingAsUser ? EyeClosedIcon : EyeOpenIcon} + leadingIconClassName={cn(SIDE_MENU_POPOVER_ITEM_ICON, IMPERSONATION_ACCENT.text)} + className={SIDE_MENU_POPOVER_ITEM_LABEL} + /> + </Form> + ); +} + function AccountMenu({ isAdmin, isImpersonating }: { isAdmin: boolean; isImpersonating: boolean }) { const [isOpen, setIsOpen] = useState(false); const navigation = useNavigation(); diff --git a/apps/webapp/app/hooks/useUser.ts b/apps/webapp/app/hooks/useUser.ts index fd9938fdb9a..aa86ba63865 100644 --- a/apps/webapp/app/hooks/useUser.ts +++ b/apps/webapp/app/hooks/useUser.ts @@ -30,9 +30,23 @@ export function useUserChanged(callback: (user: User | undefined) => void) { useChanged(useOptionalUser, callback); } +/** + * Whether the admin has switched to "view as user" for the current + * impersonation session. Display only — see `hasAdminDisplayAccess`. + */ +export function useIsViewingAsUser(matches?: UIMatch[]): boolean { + const routeMatch = useTypedMatchesData<typeof loader>({ + id: "root", + matches, + }); + + return routeMatch?.isViewingAsUser === true; +} + export function useHasAdminAccess(matches?: UIMatch[]): boolean { const user = useOptionalUser(matches); const isImpersonating = useIsImpersonating(matches); + const isViewingAsUser = useIsViewingAsUser(matches); - return Boolean(user?.admin) || isImpersonating; + return (Boolean(user?.admin) || isImpersonating) && !isViewingAsUser; } diff --git a/apps/webapp/app/models/admin.server.ts b/apps/webapp/app/models/admin.server.ts index 09811c99c2d..e93844dbaed 100644 --- a/apps/webapp/app/models/admin.server.ts +++ b/apps/webapp/app/models/admin.server.ts @@ -1,5 +1,5 @@ import { redirect } from "@remix-run/server-runtime"; -import { prisma } from "~/db.server"; +import { $replica, prisma, type PrismaClientOrTransaction } from "~/db.server"; import { logger } from "~/services/logger.server"; import type { SearchParams } from "~/routes/admin._index"; import { @@ -11,6 +11,7 @@ import { import { authenticator } from "~/services/auth.server"; import { requireUser } from "~/services/session.server"; import { extractClientIp } from "~/utils/extractClientIp.server"; +import { impersonationDestinationPath } from "~/utils/pathBuilder"; const pageSize = 20; @@ -213,7 +214,8 @@ export async function redirectWithImpersonation( request: Request, userId: string, path: string, - currentUser?: { id: string; admin: boolean } + currentUser?: { id: string; admin: boolean }, + prismaClient: PrismaClientOrTransaction = prisma ) { const user = currentUser ?? (await requireUser(request)); if (!user.admin) { @@ -224,7 +226,7 @@ export async function redirectWithImpersonation( const ipAddress = extractClientIp(xff); try { - await prisma.impersonationAuditLog.create({ + await prismaClient.impersonationAuditLog.create({ data: { action: "START", adminId: user.id, @@ -247,6 +249,87 @@ export async function redirectWithImpersonation( }); } +type ImpersonationTarget = + | { success: true; userId: string; organizationName: string } + | { success: false; reason: "org-not-found" | "no-confirmed-member" }; + +/** + * Read-only lookup of who a `/@/orgs/<slug>/…` link would impersonate: the + * first organization member who has confirmed their basic details. Writes + * nothing, so it is safe to call while only rendering the consent page. + */ +export async function findImpersonationTarget( + organizationSlug: string, + prismaClient: PrismaClientOrTransaction = $replica +): Promise<ImpersonationTarget> { + const org = await prismaClient.organization.findFirst({ + where: { + slug: organizationSlug, + deletedAt: null, + }, + select: { + title: true, + members: { + select: { + user: { + select: { + id: true, + confirmedBasicDetails: true, + }, + }, + }, + }, + }, + }); + + if (!org) { + return { success: false, reason: "org-not-found" }; + } + + const firstValidMember = org.members.find((m) => m.user.confirmedBasicDetails); + + if (!firstValidMember) { + return { success: false, reason: "no-confirmed-member" }; + } + + return { success: true, userId: firstValidMember.user.id, organizationName: org.title }; +} + +/** + * Starts impersonating the organization's first confirmed member and lands on + * the requested path with the `/@` prefix stripped. Shared by the same-origin + * loader path and the consent page's POST so there is one implementation. + * + * The destination keeps the incoming query string: both entry points are served + * at the `/@`-prefixed URL, so `request.url` carries the same search the link + * arrived with (for example the `?span=` a `/@/runs/<id>` link redirects with). + */ +export async function startImpersonation( + request: Request, + organizationSlug: string, + path: string, + currentUser: { id: string; admin: boolean }, + clients: { read: PrismaClientOrTransaction; write: PrismaClientOrTransaction } = { + read: $replica, + write: prisma, + } +) { + const target = await findImpersonationTarget(organizationSlug, clients.read); + + if (!target.success) { + logger.debug("Cannot impersonate organization", { organizationSlug, reason: target.reason }); + return clearImpersonation(request, "/admin"); + } + + return redirectWithImpersonation( + request, + target.userId, + impersonationDestinationPath(organizationSlug, path, new URL(request.url).search), + currentUser, + clients.write + ); +} + export async function clearImpersonation(request: Request, path: string) { const authUser = await authenticator.isAuthenticated(request); const targetId = await getImpersonationId(request); diff --git a/apps/webapp/app/root.tsx b/apps/webapp/app/root.tsx index 839f44b4bbc..f6c403b8c11 100644 --- a/apps/webapp/app/root.tsx +++ b/apps/webapp/app/root.tsx @@ -19,6 +19,7 @@ import { TimezoneSetter } from "./components/TimezoneSetter"; import { env } from "./env.server"; import { featuresForRequest } from "./features.server"; import { usePostHog } from "./hooks/usePostHog"; +import { getImpersonationState } from "./services/impersonation.server"; import { getUser } from "./services/session.server"; import { getTimezonePreference } from "./services/preferences/uiPreferences.server"; import { appEnvTitleTag } from "./utils"; @@ -70,6 +71,16 @@ export const loader = async ({ request }: LoaderFunctionArgs) => { }; const user = await getUser(request); + // Display-only: while impersonating, an admin can ask to see the dashboard + // the way the impersonated user sees it. Exposed from root so every route can + // read it. + // + // Resolved against the user this request authenticated as, which is the same + // condition `requireUser` applies — otherwise the flag the client reads and + // the `user.isViewingAsUser` the server computes could disagree, and the + // client-side admin UI would hide itself on a session that is not + // impersonating. + const { isViewingAsUser } = await getImpersonationState(request, user?.id); const headers = new Headers(); headers.append("Set-Cookie", await commitSession(session)); @@ -77,6 +88,7 @@ export const loader = async ({ request }: LoaderFunctionArgs) => { return typedjson( { user, + isViewingAsUser, toastMessage, posthogProjectKey, posthogUiHost, diff --git a/apps/webapp/app/routes/_app.@.orgs.$organizationSlug.$.ts b/apps/webapp/app/routes/_app.@.orgs.$organizationSlug.$.ts deleted file mode 100644 index a47deeac6aa..00000000000 --- a/apps/webapp/app/routes/_app.@.orgs.$organizationSlug.$.ts +++ /dev/null @@ -1,82 +0,0 @@ -import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; -import { redirect } from "remix-typedjson"; -import { $replica } from "~/db.server"; -import { clearImpersonation, redirectWithImpersonation } from "~/models/admin.server"; -import { env } from "~/env.server"; -import { logger } from "~/services/logger.server"; -import { requireUser } from "~/services/session.server"; -import { isSameOriginNavigation } from "~/utils/sameOriginNavigation"; - -export async function loader({ request, params }: LoaderFunctionArgs) { - const user = await requireUser(request); - - // If already impersonating, we need to clear the impersonation - if (user.isImpersonating) { - const url = new URL(request.url); - return clearImpersonation(request, url.pathname); - } - - // Only admins can impersonate - if (!user.admin) { - return redirect("/"); - } - - const path = params["*"]; - const organizationSlug = params.organizationSlug; - - logger.debug("Impersonating user", { path, organizationSlug }); - - if (!organizationSlug) { - logger.debug("Exiting impersonation mode"); - return clearImpersonation(request, "/admin"); - } - - // CSRF gate for the SET-impersonation path. Clearing impersonation - // above is benign and stays reachable without the check. - if (!isSameOriginNavigation(request, env.LOGIN_ORIGIN)) { - logger.warn("Refusing cross-site impersonation entry", { - userId: user.id, - organizationSlug, - referer: request.headers.get("referer"), - secFetchSite: request.headers.get("sec-fetch-site"), - }); - return redirect("/admin"); - } - - const org = await $replica.organization.findFirst({ - where: { - slug: organizationSlug, - deletedAt: null, - }, - select: { - members: { - select: { - user: { - select: { - id: true, - confirmedBasicDetails: true, - }, - }, - }, - }, - }, - }); - - if (!org) { - logger.debug("Organization not found", { organizationSlug }); - return clearImpersonation(request, "/admin"); - } - - const firstValidMember = org.members.find((m) => m.user.confirmedBasicDetails); - - if (!firstValidMember) { - logger.debug("No valid members found", { organizationSlug }); - return clearImpersonation(request, "/admin"); - } - - return redirectWithImpersonation( - request, - firstValidMember.user.id, - `/orgs/${organizationSlug}/${path}` - ); -} diff --git a/apps/webapp/app/routes/_app.@.orgs.$organizationSlug.$.tsx b/apps/webapp/app/routes/_app.@.orgs.$organizationSlug.$.tsx new file mode 100644 index 00000000000..2923a6fdeeb --- /dev/null +++ b/apps/webapp/app/routes/_app.@.orgs.$organizationSlug.$.tsx @@ -0,0 +1,188 @@ +import { Form } from "@remix-run/react"; +import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { redirect, typedjson, useTypedLoaderData } from "remix-typedjson"; +import { MainCenteredContainer } from "~/components/layout/AppLayout"; +import { Button } from "~/components/primitives/Buttons"; +import { Callout } from "~/components/primitives/Callout"; +import { Header1 } from "~/components/primitives/Headers"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { env } from "~/env.server"; +import { + clearImpersonation, + findImpersonationTarget, + startImpersonation, +} from "~/models/admin.server"; +import { logger } from "~/services/logger.server"; +import { requireUser } from "~/services/session.server"; +import { + impersonationConsentPostBackPath, + impersonationDestinationPath, +} from "~/utils/pathBuilder"; +import { isSameOriginNavigation } from "~/utils/sameOriginNavigation"; + +// Everything this route's loader and action touch on the server lives in +// `~/models/admin.server` on purpose: Remix only strips `loader`, `action` and +// `headers` from a route module for the browser bundle, so any other export +// here would drag server-only modules into the client build. + +export async function loader({ request, params }: LoaderFunctionArgs) { + const user = await requireUser(request); + + // If already impersonating, we need to clear the impersonation. Redirects are + // thrown, not returned, so the consent page below is the loader's only data + // shape. + if (user.isImpersonating) { + const url = new URL(request.url); + // Keep the search: `/@/runs/<id>` links redirect here carrying `?span=<spanId>`, and the + // follow-up GET builds the destination and post-back paths from it. Dropping it would land the + // admin on the run with no span selected. + throw await clearImpersonation(request, `${url.pathname}${url.search}`); + } + + // Only admins can impersonate + if (!user.admin) { + throw redirect("/"); + } + + const path = params["*"] ?? ""; + const organizationSlug = params.organizationSlug; + + logger.debug("Impersonating user", { path, organizationSlug }); + + if (!organizationSlug) { + logger.debug("Exiting impersonation mode"); + throw await clearImpersonation(request, "/admin"); + } + + // Starting impersonation is a state change, so it only happens straight away + // for an unambiguously same-origin navigation — that is what stops a + // cross-site navigation from silently starting impersonation. Links opened + // from outside the app (address bar, bookmark, a link shared elsewhere) get + // the consent page below instead, whose "Impersonate" button posts back from + // our own page and so satisfies the same check. + if (isSameOriginNavigation(request, env.LOGIN_ORIGIN)) { + throw await startImpersonation(request, organizationSlug, path, user); + } + + // Expected for any link opened outside the app (address bar, bookmark, a link + // shared elsewhere), so this is routine rather than suspicious. Only the + // referer's origin is logged — the full referer can carry another site's path + // and query string. + logger.info("Impersonation entry outside the app, showing consent page", { + userId: user.id, + organizationSlug, + refererOrigin: refererOrigin(request), + secFetchSite: request.headers.get("sec-fetch-site"), + }); + + // Read-only on purpose: nothing is written and no impersonation cookie is set + // until the admin confirms with the POST below. + const target = await findImpersonationTarget(organizationSlug); + + const search = new URL(request.url).search; + + return typedjson({ + organizationSlug, + organizationName: target.success ? target.organizationName : undefined, + destinationPath: impersonationDestinationPath(organizationSlug, path, search), + postBackPath: impersonationConsentPostBackPath(organizationSlug, path, search), + canImpersonate: target.success, + }); +} + +function refererOrigin(request: Request): string | undefined { + const referer = request.headers.get("referer"); + if (!referer) return undefined; + try { + return new URL(referer).origin; + } catch { + return undefined; + } +} + +export async function action({ request, params }: ActionFunctionArgs) { + if (request.method.toLowerCase() !== "post") { + return new Response("Method not allowed", { status: 405 }); + } + + const user = await requireUser(request); + + // Same ordering as the loader, and it matters for the same reason: while an + // impersonation cookie is set `requireUser` resolves to the *impersonated* + // user, so `user.admin` is false even for a legitimate admin. An admin who + // started impersonating in another tab and then submitted this page would + // fail the check below and be bounced to `/` with no explanation. Clear the + // impersonation first and come back to this same URL under their own + // identity, where the normal flow re-authorizes them and they can confirm. + // Never run the impersonation mutation on a request whose resolved identity + // is the impersonated user. + if (user.isImpersonating) { + const url = new URL(request.url); + // Keep the search for the same reason the loader does: the follow-up GET + // rebuilds the destination and post-back paths from it. + return clearImpersonation(request, `${url.pathname}${url.search}`); + } + + if (!user.admin) { + return redirect("/"); + } + + // The consent page posts from our own origin, so this holds. Re-applied here + // so another site cannot drive the POST either. + if (!isSameOriginNavigation(request, env.LOGIN_ORIGIN)) { + logger.warn("Refusing cross-site impersonation submission", { + userId: user.id, + organizationSlug: params.organizationSlug, + referer: request.headers.get("referer"), + secFetchSite: request.headers.get("sec-fetch-site"), + }); + return redirect("/admin"); + } + + const organizationSlug = params.organizationSlug; + + if (!organizationSlug) { + return clearImpersonation(request, "/admin"); + } + + // The consent form posts to an explicit absolute path (see + // `impersonationConsentPostBackPath`), so the organization slug, the splat + // path and the query string all arrive here intact. + return startImpersonation(request, organizationSlug, params["*"] ?? "", user); +} + +export default function Page() { + const { organizationSlug, organizationName, destinationPath, postBackPath, canImpersonate } = + useTypedLoaderData<typeof loader>(); + + return ( + <MainCenteredContainer className="max-w-88"> + <div className="flex flex-col gap-4"> + <Header1>Impersonate</Header1> + {canImpersonate ? ( + <> + <Paragraph> + Continue to impersonate a member of{" "} + <span className="text-text-bright">{organizationName ?? organizationSlug}</span> and + open <span className="text-text-bright">{destinationPath}</span>. + </Paragraph> + <Form method="post" action={postBackPath} reloadDocument> + <Button type="submit" variant="primary/medium" fullWidth shortcut={{ key: "enter" }}> + Impersonate + </Button> + </Form> + <Paragraph variant="extra-small"> + Only continue if you meant to open this link. You'll be signed in as a member of this + organization until you stop impersonating. + </Paragraph> + </> + ) : ( + <Callout variant="error"> + There's no organization <span className="text-text-bright">{organizationSlug}</span>{" "} + with a member you can impersonate. + </Callout> + )} + </div> + </MainCenteredContainer> + ); +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.custom.$dashboardId/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.custom.$dashboardId/route.tsx index 153004813a8..6f7c1d2609c 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.custom.$dashboardId/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.custom.$dashboardId/route.tsx @@ -49,7 +49,7 @@ import { getTaskIdentifiers } from "~/models/task.server"; import { MetricDashboardPresenter } from "~/presenters/v3/MetricDashboardPresenter.server"; import { QueryPresenter } from "~/presenters/v3/QueryPresenter.server"; import { removeFavoritesByUrlSubstring } from "~/services/dashboardPreferences.server"; -import { requireUser } from "~/services/session.server"; +import { hasAdminDisplayAccess, requireUser } from "~/services/session.server"; import { EnvironmentParamSchema, queryPath, @@ -98,7 +98,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { ]); // Admins and impersonating users can use EXPLAIN - const isAdmin = user.admin || user.isImpersonating; + const isAdmin = hasAdminDisplayAccess(user); // Compute widget count from dashboard layout const widgetCount = Object.keys(dashboard.layout.widgets).length; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.playground/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.playground/route.tsx index da486898d4f..ddb9f2a63eb 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.playground/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.playground/route.tsx @@ -54,6 +54,9 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { orderBy: { createdAt: "desc" }, take: 20, }), + // Raw impersonation, not `hasAdminDisplayAccess`: this list is the + // playground's region picker, so it decides which region a submitted run + // can be sent to. "View as user" only changes what is shown. new RegionsPresenter().call({ userId: user.id, projectSlug: projectParam, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx index c16037f5e10..4f5a2c1e52d 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx @@ -11,7 +11,7 @@ import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { QueryPresenter } from "~/presenters/v3/QueryPresenter.server"; import { executeQuery, getDefaultPeriod } from "~/services/queryService.server"; -import { requireUser } from "~/services/session.server"; +import { hasAdminDisplayAccess, requireUser } from "~/services/session.server"; import { EnvironmentParamSchema, queryPath } from "~/utils/pathBuilder"; import { canAccessQuery } from "~/v3/canAccessQuery.server"; import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; @@ -63,7 +63,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }); // Admins and impersonating users can use EXPLAIN - const isAdmin = user.admin || user.isImpersonating; + const isAdmin = hasAdminDisplayAccess(user); return typedjson({ defaultQuery, @@ -175,9 +175,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { } const { query, scope, explain: explainParam, period, from, to } = parsed.data; - // Only allow explain for admins/impersonating users - const isAdmin = user.admin || user.isImpersonating; - const explain = explainParam === "true" && isAdmin; + // Only allow explain for admins/impersonating users. Raw impersonation, not + // `hasAdminDisplayAccess`: this decides what the request may run, and "view as user" only changes + // what is shown — the loader is what hides the EXPLAIN control. + const explain = explainParam === "true" && (user.admin || user.isImpersonating); try { const queryResult = await executeQuery({ diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.regions/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.regions/route.tsx index 588647f22a7..f5b4e4901db 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.regions/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.regions/route.tsx @@ -54,7 +54,7 @@ import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/m import { resolveOrgIdFromSlug } from "~/models/organization.server"; import { findProjectBySlug } from "~/models/project.server"; import { type Region, RegionsPresenter } from "~/presenters/v3/RegionsPresenter.server"; -import { requireUser } from "~/services/session.server"; +import { hasAdminDisplayAccess, requireUser } from "~/services/session.server"; import { dashboardAction } from "~/services/routeBuilders/dashboardBuilder"; import { docsPath, @@ -74,7 +74,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { presenter.call({ userId: user.id, projectSlug: projectParam, - isAdmin: user.admin || user.isImpersonating, + isAdmin: hasAdminDisplayAccess(user), }) ); @@ -135,6 +135,9 @@ export const action = dashboardAction( service.call({ projectId: project.id, regionId: parsedFormData.data.regionId, + // Raw impersonation, not `hasAdminDisplayAccess`: this decides whether a restricted or + // hidden region may be set as the default, which is a capability. "View as user" only + // changes what is shown. isAdmin: user.admin || user.isImpersonating, }) ); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx index 321d6391179..ffe6dbcd038 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx @@ -117,6 +117,9 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { taskIdentifier: taskParam, environment: environment, }), + // Raw impersonation, not `hasAdminDisplayAccess`: this list is the test + // form's region picker, so it decides which region a submitted test run + // can be sent to. "View as user" only changes what is shown. new RegionsPresenter().call({ userId: user.id, projectSlug: projectParam, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam/route.tsx index c09238b6d3f..58f54fc91a9 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam/route.tsx @@ -5,7 +5,7 @@ import { DashboardAgent } from "~/components/dashboard-agent/DashboardAgent"; import { prisma } from "~/db.server"; import { updateCurrentProjectEnvironmentId } from "~/services/dashboardPreferences.server"; import { logger } from "~/services/logger.server"; -import { requireUser } from "~/services/session.server"; +import { hasAdminDisplayAccess, requireUser } from "~/services/session.server"; import { tenantContext } from "~/services/tenantContext.server"; import { selectAccessibleEnvironment } from "~/utils/environmentAccess"; import { EnvironmentParamSchema, v3ProjectPath } from "~/utils/pathBuilder"; @@ -78,10 +78,13 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { // the launcher button is hidden when it's not enabled. The org's featureFlags // came from the membership-checked project query above, so we pass them in to // avoid a second org lookup. + // Display-only, so it respects the "view as user" toggle: while that's on we + // hide the launcher an impersonated-into user wouldn't have. + const showAdminUi = hasAdminDisplayAccess(user); const hasDashboardAgentAccess = await canAccessDashboardAgent({ userId: user.id, - isAdmin: user.admin, - isImpersonating: user.isImpersonating, + isAdmin: showAdminUi && user.admin, + isImpersonating: showAdminUi && user.isImpersonating, organizationSlug, orgFeatureFlags: (project.organization.featureFlags as Record<string, unknown>) ?? {}, }); diff --git a/apps/webapp/app/routes/resources.impersonation_.view-as.ts b/apps/webapp/app/routes/resources.impersonation_.view-as.ts new file mode 100644 index 00000000000..ab1ea68baf6 --- /dev/null +++ b/apps/webapp/app/routes/resources.impersonation_.view-as.ts @@ -0,0 +1,50 @@ +import { redirect, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { env } from "~/env.server"; +import { commitImpersonationSession, setViewingAsUser } from "~/services/impersonation.server"; +import { logger } from "~/services/logger.server"; +import { requireUser } from "~/services/session.server"; +import { sanitizeRedirectPath } from "~/utils"; +import { isSameOriginNavigation } from "~/utils/sameOriginNavigation"; + +const FormSchema = z.object({ + viewAsUser: z.enum(["true", "false"]), + redirectTo: z.string().optional(), +}); + +export async function action({ request }: ActionFunctionArgs) { + if (request.method.toLowerCase() !== "post") { + return new Response("Method not allowed", { status: 405 }); + } + + const user = await requireUser(request); + + // The toggle is submitted from our own side menu, so this holds. Applied here + // for the same reason as the other impersonation routes: no other site gets to + // drive this state change. + if (!isSameOriginNavigation(request, env.LOGIN_ORIGIN)) { + logger.warn("Refusing cross-site view-as-user submission", { + userId: user.id, + secFetchSite: request.headers.get("sec-fetch-site"), + }); + return redirect("/"); + } + + const payload = Object.fromEntries(await request.formData()); + const parsed = FormSchema.safeParse(payload); + const redirectTo = sanitizeRedirectPath(parsed.success ? parsed.data.redirectTo : undefined); + + // Display-only toggle scoped to an impersonation session — outside one there + // is nothing to toggle. + if (!user.isImpersonating || !parsed.success) { + return redirect(redirectTo); + } + + const session = await setViewingAsUser(parsed.data.viewAsUser === "true", request); + + return redirect(redirectTo, { + headers: { + "Set-Cookie": await commitImpersonationSession(session), + }, + }); +} diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.bulkaction.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.bulkaction.tsx index 42631d5ff45..5bef068b994 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.bulkaction.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.bulkaction.tsx @@ -84,6 +84,12 @@ export const loader = dashboardLoader( throw new Response("Not Found", { status: 404 }); } + // Raw impersonation, not `hasAdminDisplayAccess`: this list is the bulk + // action's "override region" picker, so it decides which region a submitted + // bulk replay can re-route runs to. "View as user" only changes what is + // shown. + const isAdmin = user.admin || user.isImpersonating; + const presenter = new CreateBulkActionPresenter(); const [data, regionsResult] = await Promise.all([ presenter.call({ @@ -96,7 +102,7 @@ export const loader = dashboardLoader( new RegionsPresenter().call({ userId: user.id, projectSlug: projectParam, - isAdmin: user.admin || user.isImpersonating, + isAdmin, }) ), ]); diff --git a/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts b/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts index d6fe04be99f..e91ac2c5451 100644 --- a/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts +++ b/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts @@ -166,6 +166,11 @@ export async function loader({ request, params }: LoaderFunctionArgs) { const [payload, regionsResult] = await Promise.all([ prettyPrintPacket(run.payload, run.payloadType), + // Raw impersonation, not `hasAdminDisplayAccess`: this list is the replay + // dialog's region picker, so it decides what the submitted form can select + // — and the run's own region is returned separately, so dropping entries + // can leave the current value off the list. "View as user" only changes + // what is shown. new RegionsPresenter().call({ userId, projectSlug, diff --git a/apps/webapp/app/services/impersonation.server.ts b/apps/webapp/app/services/impersonation.server.ts index a4f3ee59c9f..e69a3fb305b 100644 --- a/apps/webapp/app/services/impersonation.server.ts +++ b/apps/webapp/app/services/impersonation.server.ts @@ -5,6 +5,7 @@ import { singleton } from "~/utils/singleton"; import { createRedisClient, type RedisClient } from "~/redis.server"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; +import { resolveImpersonationState, type ImpersonationState } from "~/utils/impersonationState"; export const impersonationSessionStorage = createCookieSessionStorage({ cookie: { @@ -18,6 +19,15 @@ export const impersonationSessionStorage = createCookieSessionStorage({ }, }); +const IMPERSONATED_USER_ID_KEY = "impersonatedUserId"; + +/** + * Display-only "view as user" flag. It lives on the impersonation cookie so it + * is scoped to the impersonation session by construction: stop impersonating + * and the flag goes with it. + */ +const VIEWING_AS_USER_KEY = "viewingAsUser"; + export function getImpersonationSession(request: Request) { return impersonationSessionStorage.getSession(request.headers.get("Cookie")); } @@ -29,13 +39,13 @@ export function commitImpersonationSession(session: Session) { export async function getImpersonationId(request: Request) { const session = await getImpersonationSession(request); - return session.get("impersonatedUserId") as string | undefined; + return session.get(IMPERSONATED_USER_ID_KEY) as string | undefined; } export async function setImpersonationId(userId: string, request: Request) { const session = await getImpersonationSession(request); - session.set("impersonatedUserId", userId); + session.set(IMPERSONATED_USER_ID_KEY, userId); return session; } @@ -43,7 +53,44 @@ export async function setImpersonationId(userId: string, request: Request) { export async function clearImpersonationId(request: Request) { const session = await getImpersonationSession(request); - session.unset("impersonatedUserId"); + session.unset(IMPERSONATED_USER_ID_KEY); + // The view-as-user flag only means anything inside an impersonation session, + // so it never outlives one. + session.unset(VIEWING_AS_USER_KEY); + + return session; +} + +/** + * The impersonation state for a request, resolved against `resolvedUserId` — the + * id the request actually authenticated as (what `getUser`/`getUserId` return). + * + * This is the one place the impersonation flags come from, so the values the + * server computes for a route and the value the root loader publishes to the + * client cannot disagree. See `resolveImpersonationState` for why the + * impersonated id has to match the resolved user rather than merely be present. + */ +export async function getImpersonationState( + request: Request, + resolvedUserId: string | undefined +): Promise<ImpersonationState> { + const session = await getImpersonationSession(request); + + return resolveImpersonationState({ + impersonatedUserId: session.get(IMPERSONATED_USER_ID_KEY), + viewingAsUser: session.get(VIEWING_AS_USER_KEY), + resolvedUserId, + }); +} + +export async function setViewingAsUser(value: boolean, request: Request) { + const session = await getImpersonationSession(request); + + if (value) { + session.set(VIEWING_AS_USER_KEY, true); + } else { + session.unset(VIEWING_AS_USER_KEY); + } return session; } diff --git a/apps/webapp/app/services/session.server.ts b/apps/webapp/app/services/session.server.ts index bdd565cf2f9..753bc7f6a17 100644 --- a/apps/webapp/app/services/session.server.ts +++ b/apps/webapp/app/services/session.server.ts @@ -3,7 +3,7 @@ import { getUserById } from "~/models/user.server"; import { sanitizeRedirectPath } from "~/utils"; import { extractClientIp } from "~/utils/extractClientIp.server"; import { authenticator } from "./auth.server"; -import { getImpersonationId } from "./impersonation.server"; +import { getImpersonationId, getImpersonationState } from "./impersonation.server"; import { logger } from "./logger.server"; import { revalidateSsoSession } from "./ssoSessionRevalidation.server"; @@ -135,7 +135,9 @@ export async function requireUser(request: Request) { throw redirect(`/login?${searchParams}`); } - const impersonationId = await getImpersonationId(request); + // Shared with the root loader so the client never reads a different answer + // than the one computed here. + const { isImpersonating, isViewingAsUser } = await getImpersonationState(request, user.id); return { id: user.id, email: user.email, @@ -148,10 +150,39 @@ export async function requireUser(request: Request) { dashboardPreferences: user.dashboardPreferences, confirmedBasicDetails: user.confirmedBasicDetails, mfaEnabledAt: user.mfaEnabledAt, - isImpersonating: !!impersonationId && impersonationId === user.id, + isImpersonating, + isViewingAsUser, }; } +/** + * Whether admin-only UI should be rendered for this user. + * + * Display only. The "view as user" toggle is cosmetic and must never widen or + * narrow a real security boundary — authorization stays on `user.admin`, the + * route builder's `authorization` block and the per-feature access checks. + * + * The rule: the toggle changes what is *shown*, never what is *permitted or + * what happens*. "Shown" is narrow — rendering and read-only listings. A badge, + * a debug tooltip, an extra table column, a control that is merely hidden while + * the handler behind it re-checks the raw flags: all fine. + * + * It does NOT extend to a value a request handler reads, nor to the option set + * of a control that submits. An option list feeding a mutation is not display: + * shrinking it changes what a submitted form is able to do, and where the + * current value is not among the remaining options it can change which value + * the form carries. Those stay on raw `user.admin || user.isImpersonating`, or + * the admin's own submissions start behaving differently — or failing — the + * moment they flip the toggle on. + */ +export function hasAdminDisplayAccess(user: { + admin: boolean; + isImpersonating: boolean; + isViewingAsUser: boolean; +}): boolean { + return (user.admin || user.isImpersonating) && !user.isViewingAsUser; +} + export async function logout(request: Request) { return redirect("/logout"); } diff --git a/apps/webapp/app/utils/impersonationPaths.test.ts b/apps/webapp/app/utils/impersonationPaths.test.ts new file mode 100644 index 00000000000..e3dba8c8241 --- /dev/null +++ b/apps/webapp/app/utils/impersonationPaths.test.ts @@ -0,0 +1,119 @@ +import { matchRoutes, resolveTo } from "@remix-run/router"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { impersonationConsentPostBackPath, impersonationDestinationPath } from "./pathBuilder"; + +// The route that renders the impersonation consent page, as the flat-route +// convention compiles `_app.@.orgs.$organizationSlug.$.tsx`. +const CONSENT_ROUTES = [ + { + id: "routes/_app", + children: [ + { + id: "routes/_app.@.orgs.$organizationSlug.$", + path: "@/orgs/:organizationSlug/*", + }, + ], + }, +]; + +/** + * What the consent route's `action` would receive for a POST to `pathname`: + * the organization slug and the splat, straight off the router. + */ +function actionParamsFor(pathname: string) { + const matches = matchRoutes(CONSENT_ROUTES, pathname); + const leaf = matches?.[matches.length - 1]; + return { + organizationSlug: leaf?.params.organizationSlug, + splat: leaf?.params["*"], + }; +} + +/** + * What `<Form method="post">` with NO `action` prop resolves to, reproducing + * `useFormAction()`: `resolveTo(".", …)` over the matched routes' contributing + * pathnames. This app does not enable `future.v3_relativeSplatPath`, so each + * contributing match supplies its `pathnameBase`. + */ +function relativeFormAction(pathname: string) { + const matches = matchRoutes(CONSENT_ROUTES, pathname) ?? []; + const contributing = matches.filter((m, i) => i === 0 || Boolean(m.route.path)); + const routePathnames = contributing.map((m) => m.pathnameBase); + return resolveTo(".", routePathnames, pathname, false).pathname; +} + +describe("impersonation consent post-back path", () => { + const slug = "acme-inc"; + // A `/@/runs/<id>` link redirects here: a deep run path plus the `?span=` + // that selects which span to open. + const splat = "projects/proj_123/env/prod/runs/run_abc"; + const search = "?span=span_xyz"; + + it("keeps the splat and the query string", () => { + expect(impersonationConsentPostBackPath(slug, splat, search)).toBe( + `/@/orgs/${slug}/${splat}${search}` + ); + expect(impersonationDestinationPath(slug, splat, search)).toBe( + `/orgs/${slug}/${splat}${search}` + ); + }); + + it("omits the query string when there isn't one", () => { + expect(impersonationConsentPostBackPath(slug, splat)).toBe(`/@/orgs/${slug}/${splat}`); + expect(impersonationDestinationPath(slug, splat)).toBe(`/orgs/${slug}/${splat}`); + }); + + it("round-trips the slug and splat back to the action", () => { + const postBackPath = impersonationConsentPostBackPath(slug, splat, search); + + expect(actionParamsFor(new URL(postBackPath, "http://localhost").pathname)).toEqual({ + organizationSlug: slug, + splat, + }); + }); + + it("the destination the action redirects to matches the one the page displayed", () => { + const postBackPath = impersonationConsentPostBackPath(slug, splat, search); + const url = new URL(postBackPath, "http://localhost"); + const params = actionParamsFor(url.pathname); + + // What `startImpersonation` builds from the action's params + request URL. + expect(impersonationDestinationPath(params.organizationSlug!, params.splat!, url.search)).toBe( + impersonationDestinationPath(slug, splat, search) + ); + }); + + // The regression this file exists for. A relative `<Form>` action drops the + // splat, so the action would start impersonation and land the admin on the + // organization root rather than the deep link the consent page promised. + it("a relative form action would drop the splat, so the path must be explicit", () => { + const consentUrl = `/@/orgs/${slug}/${splat}`; + + expect(relativeFormAction(consentUrl)).toBe(`/@/orgs/${slug}`); + expect(actionParamsFor(relativeFormAction(consentUrl))).toEqual({ + organizationSlug: slug, + splat: "", + }); + + // The explicit path does not. + expect(impersonationConsentPostBackPath(slug, splat)).toBe(consentUrl); + }); + + // Ideally this would render the route and read the emitted `action` + // attribute, but the route module imports `~/env.server`, which validates the + // full server environment at import time and so cannot be pulled into a unit + // test. Asserting on the source is the next best guard: it fails if the + // `action` prop is ever dropped, which is the whole bug. + it("the consent form names its action explicitly", () => { + const source = readFileSync( + join(__dirname, "../routes/_app.@.orgs.$organizationSlug.$.tsx"), + "utf8" + ); + + const form = source.match(/<Form\b[^>]*>/); + expect(form).not.toBeNull(); + expect(form![0]).toMatch(/action=\{postBackPath\}/); + }); +}); diff --git a/apps/webapp/app/utils/impersonationState.test.ts b/apps/webapp/app/utils/impersonationState.test.ts new file mode 100644 index 00000000000..b6fe24bf7e0 --- /dev/null +++ b/apps/webapp/app/utils/impersonationState.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import { resolveImpersonationState } from "./impersonationState"; + +const IMPERSONATED = "user_1"; +const ADMIN = "admin_1"; + +describe("resolveImpersonationState", () => { + it("reports impersonation when the cookie's id is the resolved user", () => { + expect( + resolveImpersonationState({ + impersonatedUserId: IMPERSONATED, + viewingAsUser: undefined, + resolvedUserId: IMPERSONATED, + }) + ).toEqual({ isImpersonating: true, isViewingAsUser: false }); + }); + + it("carries the view-as-user flag inside an impersonation session", () => { + expect( + resolveImpersonationState({ + impersonatedUserId: IMPERSONATED, + viewingAsUser: true, + resolvedUserId: IMPERSONATED, + }) + ).toEqual({ isImpersonating: true, isViewingAsUser: true }); + }); + + // The case the strict comparison exists for: the session falls back to the + // real admin's id when their admin role is revoked mid-session, while the + // cookie still names the impersonation target. Both flags must read false so + // the server-side values and the value published to the client agree. + it("reports neither flag when the impersonated id is not the resolved user", () => { + expect( + resolveImpersonationState({ + impersonatedUserId: IMPERSONATED, + viewingAsUser: true, + resolvedUserId: ADMIN, + }) + ).toEqual({ isImpersonating: false, isViewingAsUser: false }); + }); + + it("reports neither flag when there is no impersonated id", () => { + expect( + resolveImpersonationState({ + impersonatedUserId: undefined, + viewingAsUser: true, + resolvedUserId: IMPERSONATED, + }) + ).toEqual({ isImpersonating: false, isViewingAsUser: false }); + }); + + it("reports neither flag when there is no resolved user", () => { + expect( + resolveImpersonationState({ + impersonatedUserId: IMPERSONATED, + viewingAsUser: true, + resolvedUserId: undefined, + }) + ).toEqual({ isImpersonating: false, isViewingAsUser: false }); + }); + + it("only treats a literal true as the view-as-user flag", () => { + expect( + resolveImpersonationState({ + impersonatedUserId: IMPERSONATED, + viewingAsUser: "true", + resolvedUserId: IMPERSONATED, + }).isViewingAsUser + ).toBe(false); + }); +}); diff --git a/apps/webapp/app/utils/impersonationState.ts b/apps/webapp/app/utils/impersonationState.ts new file mode 100644 index 00000000000..2eca8efefe9 --- /dev/null +++ b/apps/webapp/app/utils/impersonationState.ts @@ -0,0 +1,47 @@ +/** + * The rule for reading impersonation state off the impersonation cookie. + * + * Kept pure and free of server-only imports so it can be unit tested directly, + * and so there is exactly one definition of "this request is impersonating" for + * every caller to share. + */ + +export type ImpersonationState = { + isImpersonating: boolean; + isViewingAsUser: boolean; +}; + +/** + * Resolves the impersonation cookie's raw contents against the identity the + * request actually authenticated as. + * + * Matching the impersonated id against `resolvedUserId` is deliberate. When an + * admin's role is revoked mid-session the session falls back to the real admin's + * id while the cookie still names the impersonation target, so "an impersonated + * id is present" and "this request is impersonating" stop meaning the same + * thing. Only the strict reading is correct there: that session is no longer + * impersonating, and so it is not viewing as the user either. + * + * Every consumer has to agree on this, or the flags computed on the server and + * the flag published to the client drift apart — the admin chrome would hide + * itself on a session that is not impersonating at all. + */ +export function resolveImpersonationState(options: { + impersonatedUserId: unknown; + viewingAsUser: unknown; + resolvedUserId: string | undefined; +}): ImpersonationState { + const { impersonatedUserId, viewingAsUser, resolvedUserId } = options; + + const isImpersonating = + typeof impersonatedUserId === "string" && + resolvedUserId !== undefined && + impersonatedUserId === resolvedUserId; + + return { + isImpersonating, + // Display only, and meaningless outside an impersonation session, so it + // never reads as on without one. + isViewingAsUser: isImpersonating && viewingAsUser === true, + }; +} diff --git a/apps/webapp/app/utils/pathBuilder.ts b/apps/webapp/app/utils/pathBuilder.ts index edd65f8bde4..d50d9e10f30 100644 --- a/apps/webapp/app/utils/pathBuilder.ts +++ b/apps/webapp/app/utils/pathBuilder.ts @@ -62,6 +62,41 @@ export function impersonate(path: string) { return `/@${path}`; } +/** + * Where a `/@/orgs/<slug>/<splat>` impersonation link lands once impersonation + * has started: the same deep link with the `/@` prefix stripped. + * + * `search` must be carried through explicitly. A `/@/runs/<id>` link redirects + * to a `v3RunSpanPath`, whose `?span=<spanId>` selects the span to open, so + * dropping it lands the admin on the run with nothing selected. + */ +export function impersonationDestinationPath( + organizationSlug: string, + splatPath: string, + search: string = "" +) { + return `/orgs/${organizationSlug}/${splatPath}${search}`; +} + +/** + * Where the impersonation consent page's form must POST back to. + * + * The form has to name this path explicitly. A `<Form>` with no `action` + * resolves to `useResolvedPath(".")`, and because this app does not enable + * `future.v3_relativeSplatPath`, that resolves to the matched route's + * `pathnameBase` — which excludes the splat. The form would post to + * `/@/orgs/<slug>`, the action would see an empty splat, and the admin would + * land on the organization root instead of the deep link the consent page just + * promised them. + */ +export function impersonationConsentPostBackPath( + organizationSlug: string, + splatPath: string, + search: string = "" +) { + return impersonate(impersonationDestinationPath(organizationSlug, splatPath, search)); +} + export function accountPath() { return `/account`; } diff --git a/apps/webapp/test/impersonationConsent.test.ts b/apps/webapp/test/impersonationConsent.test.ts new file mode 100644 index 00000000000..a481338e245 --- /dev/null +++ b/apps/webapp/test/impersonationConsent.test.ts @@ -0,0 +1,153 @@ +import { containerTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { findImpersonationTarget, startImpersonation } from "~/models/admin.server"; + +vi.setConfig({ testTimeout: 30_000 }); + +function suffix() { + return Math.random().toString(36).slice(2, 10); +} + +// A cross-site `/@/orgs/<slug>/…` navigation renders a consent page instead of +// starting impersonation, so the only work its loader does is the read-only +// target lookup. Lock that the lookup writes nothing, and that the explicit +// POST is what actually starts impersonation. +describe("impersonation consent page", () => { + containerTest( + "resolving who a link would impersonate is read-only, and the explicit POST starts impersonation", + async ({ prisma }) => { + const admin = await prisma.user.create({ + data: { + email: `admin-${suffix()}@test.local`, + authenticationMethod: "MAGIC_LINK", + admin: true, + }, + }); + + // First member has never confirmed their details, so it must be skipped. + const unconfirmed = await prisma.user.create({ + data: { + email: `unconfirmed-${suffix()}@test.local`, + authenticationMethod: "MAGIC_LINK", + confirmedBasicDetails: false, + }, + }); + const confirmed = await prisma.user.create({ + data: { + email: `confirmed-${suffix()}@test.local`, + authenticationMethod: "MAGIC_LINK", + confirmedBasicDetails: true, + }, + }); + + const slug = `acme-${suffix()}`; + const org = await prisma.organization.create({ + data: { + title: "Acme Inc", + slug, + members: { create: [{ userId: unconfirmed.id }, { userId: confirmed.id }] }, + }, + }); + + // What the consent-page loader does: look up the target, nothing else. + const target = await findImpersonationTarget(org.slug, prisma); + expect(target).toEqual({ + success: true, + userId: confirmed.id, + organizationName: "Acme Inc", + }); + + // Read-only: no impersonation was recorded by rendering the page. + expect(await prisma.impersonationAuditLog.count()).toBe(0); + + // The consent page's POST: same-origin, and it does start impersonation. + // The `?span=` is what a `/@/runs/<id>` link redirects with, so it has to + // survive to the destination or the run opens with no span selected. + const response = await startImpersonation( + new Request( + `http://localhost:3030/@/orgs/${org.slug}/projects/p/runs/run_123?span=span_abc`, + { + method: "POST", + headers: { "sec-fetch-site": "same-origin" }, + } + ), + org.slug, + "projects/p/runs/run_123", + { id: admin.id, admin: true }, + { read: prisma, write: prisma } + ); + + expect(response.status).toBe(302); + // The splat path and query string are preserved, `/@` prefix stripped. + expect(response.headers.get("location")).toBe( + `/orgs/${org.slug}/projects/p/runs/run_123?span=span_abc` + ); + expect(response.headers.get("set-cookie")).toContain("__impersonate="); + + const auditLogs = await prisma.impersonationAuditLog.findMany(); + expect(auditLogs).toHaveLength(1); + expect(auditLogs[0]).toMatchObject({ + action: "START", + adminId: admin.id, + targetId: confirmed.id, + }); + } + ); + + containerTest("an unknown organization slug cannot be impersonated", async ({ prisma }) => { + expect(await findImpersonationTarget(`missing-${suffix()}`, prisma)).toEqual({ + success: false, + reason: "org-not-found", + }); + }); + + containerTest( + "an organization with no confirmed members cannot be impersonated", + async ({ prisma }) => { + const user = await prisma.user.create({ + data: { + email: `unconfirmed-${suffix()}@test.local`, + authenticationMethod: "MAGIC_LINK", + confirmedBasicDetails: false, + }, + }); + + const org = await prisma.organization.create({ + data: { + title: "Nobody Inc", + slug: `nobody-${suffix()}`, + members: { create: [{ userId: user.id }] }, + }, + }); + + expect(await findImpersonationTarget(org.slug, prisma)).toEqual({ + success: false, + reason: "no-confirmed-member", + }); + } + ); + + containerTest("a deleted organization cannot be impersonated", async ({ prisma }) => { + const user = await prisma.user.create({ + data: { + email: `confirmed-${suffix()}@test.local`, + authenticationMethod: "MAGIC_LINK", + confirmedBasicDetails: true, + }, + }); + + const org = await prisma.organization.create({ + data: { + title: "Gone Inc", + slug: `gone-${suffix()}`, + deletedAt: new Date(), + members: { create: [{ userId: user.id }] }, + }, + }); + + expect(await findImpersonationTarget(org.slug, prisma)).toEqual({ + success: false, + reason: "org-not-found", + }); + }); +}); diff --git a/apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts b/apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts index 8d844f550b3..f6597980f39 100644 --- a/apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts +++ b/apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts @@ -36,7 +36,12 @@ vi.setConfig({ testTimeout: 120_000, hookTimeout: 120_000 }); // `authUser`, `resolvedEnv`, `logDetail`, `project`, `environment` feed the mocked fix-orthogonal deps. const cp = vi.hoisted(() => ({ client: undefined as any })); const router = vi.hoisted(() => ({ store: undefined as any })); -const authUser = vi.hoisted(() => ({ id: "user_rrg_guard", admin: false, isImpersonating: false })); +const authUser = vi.hoisted(() => ({ + id: "user_rrg_guard", + admin: false, + isImpersonating: false, + isViewingAsUser: false, +})); const resolved = vi.hoisted(() => ({ authEnv: undefined as any, env: undefined as any, @@ -104,7 +109,10 @@ vi.mock("~/v3/runStore.server", () => ({ ), })); -// Auth/session (orthogonal): fixed user id. +// Auth/session (orthogonal): fixed user id. This factory replaces the whole module, so it has to +// return every export the route graph under test actually reaches — accessing one it does not +// define throws. Keep it to that set; importing the original would evaluate session.server's +// server-only import graph, which this test deliberately keeps out. vi.mock("~/services/session.server", () => ({ requireUserId: async () => authUser.id, requireUser: async () => authUser, diff --git a/apps/webapp/test/viewAsUser.test.ts b/apps/webapp/test/viewAsUser.test.ts new file mode 100644 index 00000000000..d21d53f6bc6 --- /dev/null +++ b/apps/webapp/test/viewAsUser.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from "vitest"; +import { + clearImpersonationId, + commitImpersonationSession, + getImpersonationId, + getImpersonationState, + setImpersonationId, + setViewingAsUser, +} from "~/services/impersonation.server"; +import { hasAdminDisplayAccess } from "~/services/session.server"; +import type { Session } from "@remix-run/node"; + +// `commitSession` returns a full Set-Cookie value; a request only needs the +// name=value pair back. +async function requestWith(session: Session) { + const setCookie = await commitImpersonationSession(session); + return new Request("http://localhost:3030/orgs/acme", { + headers: { Cookie: setCookie.split(";")[0] }, + }); +} + +// Reads the flag the way a request does: against the user the request resolved +// as. Every fixture below impersonates `user_1`. +async function readViewingAsUser(request: Request) { + return (await getImpersonationState(request, "user_1")).isViewingAsUser; +} + +// The "view as user" flag rides on the impersonation cookie, so it is scoped to +// the impersonation session by construction. +describe("view as user flag", () => { + it("is off when there is no impersonation cookie", async () => { + expect(await readViewingAsUser(new Request("http://localhost:3030/orgs/acme"))).toBe(false); + }); + + it("can be turned on and back off within an impersonation session", async () => { + const start = new Request("http://localhost:3030/orgs/acme"); + + const impersonating = await requestWith(await setImpersonationId("user_1", start)); + expect(await getImpersonationId(impersonating)).toBe("user_1"); + expect(await readViewingAsUser(impersonating)).toBe(false); + + const viewingAsUser = await requestWith(await setViewingAsUser(true, impersonating)); + expect(await getImpersonationId(viewingAsUser)).toBe("user_1"); + expect(await readViewingAsUser(viewingAsUser)).toBe(true); + + const showingAdminUi = await requestWith(await setViewingAsUser(false, viewingAsUser)); + expect(await getImpersonationId(showingAdminUi)).toBe("user_1"); + expect(await readViewingAsUser(showingAdminUi)).toBe(false); + }); + + it("is dropped when impersonation is cleared", async () => { + const start = new Request("http://localhost:3030/orgs/acme"); + const impersonating = await requestWith(await setImpersonationId("user_1", start)); + const viewingAsUser = await requestWith(await setViewingAsUser(true, impersonating)); + + const cleared = await requestWith(await clearImpersonationId(viewingAsUser)); + + expect(await getImpersonationId(cleared)).toBeUndefined(); + expect(await readViewingAsUser(cleared)).toBe(false); + }); + + it("never reads as on without an impersonated user", async () => { + const start = new Request("http://localhost:3030/orgs/acme"); + // Set the flag with no impersonation in progress — it must not read back on. + const flagOnly = await requestWith(await setViewingAsUser(true, start)); + + expect(await readViewingAsUser(flagOnly)).toBe(false); + }); + + it("is off once the impersonated user is not who the request resolved as", async () => { + const start = new Request("http://localhost:3030/orgs/acme"); + const impersonating = await requestWith(await setImpersonationId("user_1", start)); + const viewing = await requestWith(await setViewingAsUser(true, impersonating)); + + // An admin who loses the admin role mid-session resolves back to their own + // id while the cookie still names the target. That session is not + // impersonating any more, so it is not viewing as the user either. + const state = await getImpersonationState(viewing, "admin_1"); + + expect(state.isImpersonating).toBe(false); + expect(state.isViewingAsUser).toBe(false); + }); +}); + +describe("hasAdminDisplayAccess", () => { + it("shows admin UI to admins and to impersonating sessions", () => { + expect( + hasAdminDisplayAccess({ admin: true, isImpersonating: false, isViewingAsUser: false }) + ).toBe(true); + expect( + hasAdminDisplayAccess({ admin: false, isImpersonating: true, isViewingAsUser: false }) + ).toBe(true); + }); + + it("hides admin UI to everyone else", () => { + expect( + hasAdminDisplayAccess({ admin: false, isImpersonating: false, isViewingAsUser: false }) + ).toBe(false); + }); + + it("hides admin UI while viewing as the user", () => { + expect( + hasAdminDisplayAccess({ admin: true, isImpersonating: true, isViewingAsUser: true }) + ).toBe(false); + expect( + hasAdminDisplayAccess({ admin: false, isImpersonating: true, isViewingAsUser: true }) + ).toBe(false); + }); +}); From efcb89ac26984aac8e98f9c6ccb9a902c0483881 Mon Sep 17 00:00:00 2001 From: Wes Mason <wes@1stvamp.org> Date: Fri, 31 Jul 2026 09:44:16 +0100 Subject: [PATCH 44/67] fix(webapp): add hasAdminDisplayAccess to the env param test mock (#4430) `test/envParamRoute.ownership.test.ts` fails on main: 3 of its 4 tests throw ``` Error: [vitest] No "hasAdminDisplayAccess" export is defined on the "~/services/session.server" mock. Did you forget to return it from "vi.mock"? ``` #4421 added a `hasAdminDisplayAccess(user)` call to the `env.$envParam` loader, and the test's `vi.mock` of `session.server` only returns `requireUser`, so the call blows up. Both changes were green in their own PR and only conflict once merged together, which is why nobody caught it. The mock now mirrors the real implementation rather than returning a constant, so it stays correct if the test's user fixture is ever varied. No assertions were changed: the tests were right, the mock was stale. Worth flagging separately: no workflow runs on push to main, so this has been red since #4421 landed without showing up anywhere. Every PR opened since has inherited the failure. --- apps/webapp/test/envParamRoute.ownership.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/webapp/test/envParamRoute.ownership.test.ts b/apps/webapp/test/envParamRoute.ownership.test.ts index 624ef711352..399ecdb02eb 100644 --- a/apps/webapp/test/envParamRoute.ownership.test.ts +++ b/apps/webapp/test/envParamRoute.ownership.test.ts @@ -17,6 +17,11 @@ vi.mock("~/db.server", () => ({ vi.mock("~/services/session.server", () => ({ requireUser: async () => mocks.user, + hasAdminDisplayAccess: (user: { + admin: boolean; + isImpersonating: boolean; + isViewingAsUser?: boolean; + }) => (user.admin || user.isImpersonating) && !user.isViewingAsUser, })); vi.mock("~/services/dashboardPreferences.server", () => ({ From 17d849b2d627a23813c2bdecdf5ebc77dbd5ee95 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <209825114+claude[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2026 11:54:42 +0100 Subject: [PATCH 45/67] feat(cli): expose region option on the MCP trigger_task tool (#4439) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit <!-- ccr-slack-attribution --> _Requested by **Eric Allam** · [Slack thread](https://triggerdotdev.slack.com/archives/C0BEM9Z73TM/p1785491472104199)_ ## Checklist - [ ] I have followed every step in the [contributing guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md) - [x] The PR title follows the convention. - [ ] I ran and tested the code works --- ## Testing Static checks only, all clean: - `pnpm run typecheck --filter trigger.dev` - `pnpm run format` - `pnpm run lint` No live task was triggered against a running project, so the "ran and tested" box above is left unchecked. --- ## Changelog **Before:** triggering a task through the MCP server always ran it in the project's default region. There was no way to pick one. **After:** the `trigger_task` tool accepts an optional `region` option, so you can choose the region a run executes in. **How:** `region: z.string().optional()` was added to `TriggerTaskInput.options` in `packages/cli-v3/src/mcp/schemas.ts`. No call-site change was needed — `tools/tasks.ts` passes `options` through verbatim, and `TriggerTaskRequestBody.options.region` already existed. The tool description in `docs/mcp-tools.mdx` gained a matching line, and a patch changeset is included. There is no batch-trigger MCP tool, so there is no sibling tool to mirror this change on. --- ## Screenshots N/A — no UI changes. Co-authored-by: Claude <noreply@anthropic.com> --- .changeset/mcp-trigger-task-region.md | 5 +++++ docs/mcp-tools.mdx | 2 +- packages/cli-v3/src/mcp/schemas.ts | 6 ++++++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 .changeset/mcp-trigger-task-region.md diff --git a/.changeset/mcp-trigger-task-region.md b/.changeset/mcp-trigger-task-region.md new file mode 100644 index 00000000000..56e362eb1fb --- /dev/null +++ b/.changeset/mcp-trigger-task-region.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +You can now choose the region a run executes in when triggering a task through the MCP server. diff --git a/docs/mcp-tools.mdx b/docs/mcp-tools.mdx index 9a920274fba..d35a733b091 100644 --- a/docs/mcp-tools.mdx +++ b/docs/mcp-tools.mdx @@ -61,7 +61,7 @@ Get the current worker for a project, including the worker version, SDK version, ### trigger_task -Trigger a task to run with a specific payload. You can add a delay, set tags, configure retries, choose a machine size, set a TTL, or use an idempotency key. +Trigger a task to run with a specific payload. You can add a delay, set tags, configure retries, choose a machine size, pick a region, set a TTL, or use an idempotency key. **Example usage:** - `"Run the email-notification task"` diff --git a/packages/cli-v3/src/mcp/schemas.ts b/packages/cli-v3/src/mcp/schemas.ts index 64a03438028..c711735537a 100644 --- a/packages/cli-v3/src/mcp/schemas.ts +++ b/packages/cli-v3/src/mcp/schemas.ts @@ -100,6 +100,12 @@ export const TriggerTaskInput = CommonProjectsInput.extend({ .number() .describe("The maximum duration in seconds of the task run") .optional(), + region: z + .string() + .describe( + "The region to run the task in, overriding the default region set for the project. Available regions are listed on the Regions page in the dashboard, and this has no effect in the dev environment" + ) + .optional(), tags: z .array(z.string()) .describe( From c72ebf908447a66cfc85bf35b987f4caf5639303 Mon Sep 17 00:00:00 2001 From: Eric Allam <eric@trigger.dev> Date: Fri, 31 Jul 2026 11:55:25 +0100 Subject: [PATCH 46/67] fix(webapp,run-engine): stop batchTriggerAndWait hanging when item streaming never completes (#4397) ## Summary `batchTriggerAndWait()` could leave a parent run waiting forever. The 2-phase batch API blocks the parent on the batch's waitpoint as soon as the batch is created, but the batch is only sealed at the end of item streaming. If streaming never completed, nothing sealed the batch, nothing completed the waitpoint, and the parent stayed suspended with no timeout and no way to recover. Supersedes #4016, which added the reaper alone. ## Fix Admission for item streaming was being decided twice. Batch creation passes its own rate limiter, which fixes `expectedCount` and blocks the parent, and then the item stream had to pass the general API limiter as well, competing with unrelated traffic. A second limiter could therefore veto work the first had already committed the parent to. Creation now mints a bounded grant that the item stream spends, so an admitted batch can finish streaming. The grant is capped per batch rather than exempting the path, and every failure mode (no grant, spent grant, unreachable store) falls back to the normal limiter. That makes stranding much rarer but not impossible, since a request timeout or a crash can still end streaming for good. So a seal-timeout reaper aborts any batch still unsealed after `BATCH_SEAL_TIMEOUT_MS` and completes the parent's waitpoint with an error, letting `batchTriggerAndWait()` reject instead of hang. It is race-safe against a late seal, and it is only scheduled for batches that actually block a parent, so fire-and-forget batches cost nothing. Finally, the batches page used to report "Batch completion checked." for these batches while doing nothing, because the completion path returns early on an unsealed batch. It now says the batch cannot be resumed. Rate limiting is no longer the reason a batch strands, so the reaper's default stays at 30 minutes, comfortably above the SDK's worst-case stream-retry budget. ## Verification Unit and container tests cover the grant cap, the bypass ordering (it runs after the authorization check, so it can never skip authentication), and the reaper's abort, seal race, idempotency, and no-waitpoint cases. Also verified end-to-end against a running stack. With the general limit exhausted, batch creation and other API calls returned 429 while a granted batch still streamed and sealed; an ungranted batch id was rate limited rather than bypassed; and the grant cut off exactly at its configured attempt count. Reproducing the stranded state on a real parent run, the batch was aborted at the timeout, the waitpoint completed with an error, and the parent resumed and finished instead of hanging. A parentless batch left unsealed was untouched well past the reaper window. ## Verified against deployed runs The reaper was proven end to end with a real deployed run (locally-run supervisor, containerised run) and a real network fault, rather than a simulated one: toxiproxy severs the phase 2 item stream mid-flight so every SDK stream retry genuinely fails, while phase 1 still succeeds. Only the batch calls traverse the fault, so control-plane traffic is untouched. The reproduction is the shape that actually strands a parent: the task catches the `BatchTriggerError` the SDK throws and carries on, so the phase 1 block outlives the thrown error and the parent hangs at its next suspension point. With the reaper disabled, the parent sat in `EXECUTING_WITH_WAITPOINTS` for over 24 minutes holding two blockers, and stayed stuck across a full infrastructure restart: ``` type | status | has_timeout BATCH | PENDING | f <- orphan, completedAfter NULL DATETIME | COMPLETED | t <- the wait already elapsed ``` With the reaper enabled the same task under the same fault completed in about 75 seconds with zero blockers left, the batch `ABORTED`, and its waitpoint completed carrying the error. Two conditions are required to observe this at all, which is worth knowing for any future test: the run must be deployed rather than `trigger dev` (dev runs execute in process and finish while still holding blocker rows), and the wait after the caught error must exceed the checkpoint threshold, or it is served in process and never suspends. ### Why completing the batch waitpoint is sufficient `batchTriggerAndWait` runs create, then stream, then wait. A phase 2 failure throws before the wait is ever reached, and the reaper only fires on an unsealed batch, so the parent is never suspended awaiting the batch when it runs. The parent therefore does not need a synthetic result, only to stop being blocked. Note this reasoning depends on that ordering: if the wait were ever reached with an unsealed batch, completing the batch waitpoint alone would not settle the caller. ## Follow-ups - Batches stranded before this ships still need a one-off recovery; the reaper only schedules at creation time. - That same property leaves a gap if the process dies between creating the batch and scheduling the job. A periodic sweep would close it, but wants a supporting index. - When a partially streamed batch aborts, children already enqueued keep running while the parent fails. Left as-is deliberately, since cancelling triggered work is a bigger semantic call. --- .../batch-item-streaming-rate-limit.md | 6 + apps/webapp/app/env.server.ts | 14 + ...urces.batches.$batchId.check-completion.ts | 14 + .../concerns/batchStreamGrants.server.ts | 105 ++++++ .../batchStreamGrantsInstance.server.ts | 20 ++ .../runEngine/services/createBatch.server.ts | 9 + .../app/services/apiRateLimit.server.ts | 30 ++ ...authorizationRateLimitMiddleware.server.ts | 42 ++- .../v3/runOpsMigration/unblockRouteCatalog.ts | 6 + .../authorizationRateLimitMiddleware.test.ts | 20 +- ...orizationRateLimitMiddlewareBypass.test.ts | 92 +++++ apps/webapp/test/batchStreamGrants.test.ts | 102 ++++++ .../run-engine/src/engine/index.ts | 25 ++ .../src/engine/systems/batchSystem.test.ts | 8 + .../src/engine/systems/batchSystem.ts | 148 +++++++- .../src/engine/tests/batchTwoPhase.test.ts | 320 ++++++++++++++++++ .../run-engine/src/engine/workerCatalog.ts | 6 + 17 files changed, 934 insertions(+), 33 deletions(-) create mode 100644 .server-changes/batch-item-streaming-rate-limit.md create mode 100644 apps/webapp/app/runEngine/concerns/batchStreamGrants.server.ts create mode 100644 apps/webapp/app/runEngine/concerns/batchStreamGrantsInstance.server.ts create mode 100644 apps/webapp/test/authorizationRateLimitMiddlewareBypass.test.ts create mode 100644 apps/webapp/test/batchStreamGrants.test.ts diff --git a/.server-changes/batch-item-streaming-rate-limit.md b/.server-changes/batch-item-streaming-rate-limit.md new file mode 100644 index 00000000000..f817195e9b1 --- /dev/null +++ b/.server-changes/batch-item-streaming-rate-limit.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Batch triggers no longer fail to start their runs when an environment is under heavy API load. If a batch still can't finish being created, `batchTriggerAndWait` now fails with an error instead of leaving the parent run waiting forever, and the batches page says so rather than reporting that it resumed. diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 24ed0833d8e..b371d5994e9 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -884,6 +884,20 @@ const EnvironmentSchema = z BATCH_RATE_LIMIT_MAX: z.coerce.number().int().default(1200), BATCH_RATE_LIMIT_REFILL_INTERVAL: z.string().default("10s"), BATCH_CONCURRENCY_LIMIT_DEFAULT: z.coerce.number().int().default(5), + /** + * How long a created batch may remain unsealed before the seal-timeout reaper + * aborts it and resumes any blocked parent with an error. Must exceed the SDK's + * worst-case stream-retry budget (maxAttempts x server request timeout). + * Doubles as the TTL of the phase 2 streaming grant, so the grant and the reaper + * always agree on how long a batch is allowed to be sealing. + */ + BATCH_SEAL_TIMEOUT_MS: z.coerce.number().int().positive().default(1_800_000), + /** + * Number of phase 2 (`POST /api/v3/batches/:id/items`) requests a created batch is + * granted, exempt from the general API rate limit. Sized above the SDK's stream + * maxAttempts so a batch admitted by the batch limiter can always finish streaming. + */ + BATCH_STREAM_GRANT_ATTEMPTS: z.coerce.number().int().positive().default(10), REALTIME_STREAM_VERSION: z.enum(["v1", "v2"]).default("v1"), REALTIME_STREAM_MAX_LENGTH: z.coerce.number().int().default(1000), diff --git a/apps/webapp/app/routes/resources.batches.$batchId.check-completion.ts b/apps/webapp/app/routes/resources.batches.$batchId.check-completion.ts index fffa0d09989..2d1c6bc702b 100644 --- a/apps/webapp/app/routes/resources.batches.$batchId.check-completion.ts +++ b/apps/webapp/app/routes/resources.batches.$batchId.check-completion.ts @@ -42,6 +42,20 @@ export const action: ActionFunction = async ({ request, params }) => { return redirectWithErrorMessage(safeRedirectUrl, request, "Batch not found"); } + const batch = await runStore.findBatchTaskRunById(ownedBatchRunId); + + if (!batch) { + return redirectWithErrorMessage(safeRedirectUrl, request, "Batch not found"); + } + + if (!batch.sealed) { + return redirectWithErrorMessage( + safeRedirectUrl, + request, + "This batch was never finished being created, so it can't be resumed. Please get in touch and we'll recover it for you." + ); + } + try { // v3 (engine V1) is retired; finalize the batch through the v2 completion path (no-op if not ready). await tryCompleteBatchV3(ownedBatchRunId, prisma, true); diff --git a/apps/webapp/app/runEngine/concerns/batchStreamGrants.server.ts b/apps/webapp/app/runEngine/concerns/batchStreamGrants.server.ts new file mode 100644 index 00000000000..17fc7324744 --- /dev/null +++ b/apps/webapp/app/runEngine/concerns/batchStreamGrants.server.ts @@ -0,0 +1,105 @@ +import { createRedisClient, type RedisClient, type RedisWithClusterOptions } from "~/redis.server"; +import { logger } from "~/services/logger.server"; + +export type BatchStreamGrantsOptions = { + redis: RedisWithClusterOptions; + /** How many phase 2 requests a created batch is allowed. */ + attempts: number; + /** How long the grant survives, matching how long a batch may legitimately be sealing. */ + ttlMs: number; +}; + +const KEY_PREFIX = "batch-stream-grant:"; + +/** + * Admission for phase 2 of the 2-phase batch API. + * + * Phase 1 (`POST /api/v3/batches`) already passes its own batch rate limiter, which fixes + * the batch's `expectedCount` and blocks the parent run on the batch's waitpoint. Phase 2 + * (`POST /api/v3/batches/:id/items`) is the only thing that can seal that batch, so having + * the general API limiter reject it strands the batch and the parent with it. + * + * Phase 1 therefore mints a bounded grant, and phase 2 spends it to bypass the general + * limiter. Admission stays a single decision made in phase 1, but the bypass is capped at + * `attempts` requests per batch rather than being unconditional. + */ +export class BatchStreamGrants { + private readonly redis: RedisClient; + + constructor(private readonly options: BatchStreamGrantsOptions) { + this.redis = createRedisClient("batchStreamGrants", options.redis); + this.#registerCommands(); + } + + /** + * Grant a newly created batch its phase 2 budget. Never throws: a batch that fails to get + * a grant still works, it just falls back to the general rate limiter for streaming. + */ + async mint(environmentId: string, batchId: string): Promise<void> { + try { + await this.redis.set( + this.#key(environmentId, batchId), + this.options.attempts, + "PX", + this.options.ttlMs + ); + } catch (error) { + logger.warn("BatchStreamGrants: failed to mint grant", { + batchId, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + /** + * Consume one phase 2 request from the batch's grant. + * + * Returns false when there is no grant, when the budget is spent, or when Redis is + * unreachable, so the caller falls back to the general rate limiter rather than opening + * an unbounded bypass. + */ + async spend(environmentId: string, batchId: string): Promise<boolean> { + try { + // @ts-expect-error - Custom command defined via defineCommand + const remaining = (await this.redis.spendBatchStreamGrant( + this.#key(environmentId, batchId) + )) as number; + + return remaining >= 0; + } catch (error) { + logger.warn("BatchStreamGrants: failed to spend grant", { + batchId, + error: error instanceof Error ? error.message : String(error), + }); + + return false; + } + } + + async quit(): Promise<void> { + await this.redis.quit(); + } + + /** + * Scoped to the environment as well as the batch, so a caller authenticated against a + * different environment can never spend this batch's grant even if they know its id. + */ + #key(environmentId: string, batchId: string): string { + return `${KEY_PREFIX}${environmentId}:${batchId}`; + } + + #registerCommands(): void { + this.redis.defineCommand("spendBatchStreamGrant", { + numberOfKeys: 1, + lua: ` +local remaining = tonumber(redis.call('GET', KEYS[1])) + +if not remaining or remaining <= 0 then + return -1 +end + +return redis.call('DECR', KEYS[1]) + `, + }); + } +} diff --git a/apps/webapp/app/runEngine/concerns/batchStreamGrantsInstance.server.ts b/apps/webapp/app/runEngine/concerns/batchStreamGrantsInstance.server.ts new file mode 100644 index 00000000000..1d79dc46674 --- /dev/null +++ b/apps/webapp/app/runEngine/concerns/batchStreamGrantsInstance.server.ts @@ -0,0 +1,20 @@ +import { env } from "~/env.server"; +import { singleton } from "~/utils/singleton"; +import { BatchStreamGrants } from "./batchStreamGrants.server"; + +export const batchStreamGrants = singleton( + "batchStreamGrants", + () => + new BatchStreamGrants({ + redis: { + port: env.RATE_LIMIT_REDIS_PORT, + host: env.RATE_LIMIT_REDIS_HOST, + username: env.RATE_LIMIT_REDIS_USERNAME, + password: env.RATE_LIMIT_REDIS_PASSWORD, + tlsDisabled: env.RATE_LIMIT_REDIS_TLS_DISABLED === "true", + clusterMode: env.RATE_LIMIT_REDIS_CLUSTER_MODE_ENABLED === "1", + }, + attempts: env.BATCH_STREAM_GRANT_ATTEMPTS, + ttlMs: env.BATCH_SEAL_TIMEOUT_MS, + }) +); diff --git a/apps/webapp/app/runEngine/services/createBatch.server.ts b/apps/webapp/app/runEngine/services/createBatch.server.ts index 0095c48f2b5..0289e68e2c7 100644 --- a/apps/webapp/app/runEngine/services/createBatch.server.ts +++ b/apps/webapp/app/runEngine/services/createBatch.server.ts @@ -4,6 +4,8 @@ import { RunId } from "@trigger.dev/core/v3/isomorphic"; import { type BatchTaskRun, Prisma } from "@trigger.dev/database"; import { Evt } from "evt"; import { prisma, type PrismaClientOrTransaction } from "~/db.server"; +import { env } from "~/env.server"; +import { batchStreamGrants } from "../concerns/batchStreamGrantsInstance.server"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; @@ -120,8 +122,15 @@ export class CreateBatchService extends WithRunEngine { this.onBatchTaskRunCreated.post(batch); + await batchStreamGrants.mint(environment.id, friendlyId); + // Block parent run if this is a batchTriggerAndWait if (body.parentRunId && body.resumeParentOnCompletion) { + await this._engine.scheduleExpireBatch({ + batchId: batch.id, + availableAt: new Date(Date.now() + env.BATCH_SEAL_TIMEOUT_MS), + }); + await this._engine.blockRunWithCreatedBatch({ runId: RunId.fromFriendlyId(body.parentRunId), batchId: batch.id, diff --git a/apps/webapp/app/services/apiRateLimit.server.ts b/apps/webapp/app/services/apiRateLimit.server.ts index 837d3ee5895..1b8d8a3ed1f 100644 --- a/apps/webapp/app/services/apiRateLimit.server.ts +++ b/apps/webapp/app/services/apiRateLimit.server.ts @@ -1,8 +1,12 @@ +import { tryCatch } from "@trigger.dev/core/v3"; import { env } from "~/env.server"; +import { batchStreamGrants } from "~/runEngine/concerns/batchStreamGrantsInstance.server"; import { authenticateAuthorizationHeader } from "./apiAuth.server"; import { authorizationRateLimitMiddleware } from "./authorizationRateLimitMiddleware.server"; import type { Duration } from "./rateLimiter.server"; +const BATCH_STREAM_ITEMS_PATH = /^\/api\/v3\/batches\/([^/]+)\/items$/; + export const apiRateLimiter = authorizationRateLimitMiddleware({ redis: { port: env.RATE_LIMIT_REDIS_PORT, @@ -75,6 +79,32 @@ export const apiRateLimiter = authorizationRateLimitMiddleware({ /^\/api\/v2\/packets\//, /^\/api\/v1\/sessions\/[^/]+\/snapshot-url$/, ], + bypass: async (req) => { + const match = BATCH_STREAM_ITEMS_PATH.exec(req.path); + + if (!match) { + return false; + } + + const batchFriendlyId = match[1]; + const authorizationValue = req.headers.authorization; + + if (!batchFriendlyId || !authorizationValue) { + return false; + } + + const [authError, authenticated] = await tryCatch( + authenticateAuthorizationHeader(authorizationValue, { + allowPublicKey: true, + }) + ); + + if (authError || !authenticated || !authenticated.ok) { + return false; + } + + return batchStreamGrants.spend(authenticated.environment.id, batchFriendlyId); + }, log: { rejections: env.API_RATE_LIMIT_REJECTION_LOGS_ENABLED === "1", requests: env.API_RATE_LIMIT_REQUEST_LOGS_ENABLED === "1", diff --git a/apps/webapp/app/services/authorizationRateLimitMiddleware.server.ts b/apps/webapp/app/services/authorizationRateLimitMiddleware.server.ts index e6d4d3b1f5e..5fcd6eb5450 100644 --- a/apps/webapp/app/services/authorizationRateLimitMiddleware.server.ts +++ b/apps/webapp/app/services/authorizationRateLimitMiddleware.server.ts @@ -5,7 +5,6 @@ import { Ratelimit } from "@upstash/ratelimit"; import type { Request as ExpressRequest, Response as ExpressResponse, NextFunction } from "express"; import { createHash } from "node:crypto"; import { z } from "zod"; -import { env } from "~/env.server"; import type { RedisWithClusterOptions } from "~/redis.server"; import { logger } from "./logger.server"; import type { Duration, Limiter } from "./rateLimiter.server"; @@ -56,10 +55,17 @@ export type RateLimiterConfig = z.infer<typeof RateLimiterConfig>; type LimitConfigOverrideFunction = (authorizationValue: string) => Promise<unknown>; type Options = { - redis?: RedisWithClusterOptions; + redis: RedisWithClusterOptions; keyPrefix: string; pathMatchers: (RegExp | string)[]; pathWhiteList?: (RegExp | string)[]; + /** + * Escape hatch for requests that can only be admitted by consulting state, rather than by + * matching a path. Runs after the authorization header check, so an unauthenticated + * request is still rejected, and only skips the rate limit itself. Must not throw: a + * bypass that cannot decide should return false and let the limiter apply. + */ + bypass?: (req: ExpressRequest) => Promise<boolean>; defaultLimiter: RateLimiterConfig; limiterConfigOverride?: LimitConfigOverrideFunction; limiterCache?: { @@ -151,6 +157,7 @@ export function authorizationRateLimitMiddleware({ defaultLimiter, pathMatchers, pathWhiteList = [], + bypass, log = { rejections: true, requests: true, @@ -176,16 +183,7 @@ export function authorizationRateLimitMiddleware({ }), }); - const redisClient = createRedisRateLimitClient( - redis ?? { - port: env.RATE_LIMIT_REDIS_PORT, - host: env.RATE_LIMIT_REDIS_HOST, - username: env.RATE_LIMIT_REDIS_USERNAME, - password: env.RATE_LIMIT_REDIS_PASSWORD, - tlsDisabled: env.RATE_LIMIT_REDIS_TLS_DISABLED === "true", - clusterMode: env.RATE_LIMIT_REDIS_CLUSTER_MODE_ENABLED === "1", - } - ); + const redisClient = createRedisRateLimitClient(redis); return async (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => { if (log.requests) { @@ -247,6 +245,26 @@ export function authorizationRateLimitMiddleware({ ); } + if (bypass) { + let bypassed = false; + + try { + bypassed = await bypass(req); + } catch (error) { + logger.warn(`RateLimiter (${keyPrefix}): bypass threw, applying the limit`, { + path: req.path, + error: error instanceof Error ? error.message : String(error), + }); + } + + if (bypassed) { + if (log.requests) { + logger.info(`RateLimiter (${keyPrefix}): bypassed ${req.path}`); + } + return next(); + } + } + const hash = createHash("sha256"); hash.update(authorizationValue); const hashedAuthorizationValue = hash.digest("hex"); diff --git a/apps/webapp/app/v3/runOpsMigration/unblockRouteCatalog.ts b/apps/webapp/app/v3/runOpsMigration/unblockRouteCatalog.ts index 3296569ce0b..7d7f9d6f536 100644 --- a/apps/webapp/app/v3/runOpsMigration/unblockRouteCatalog.ts +++ b/apps/webapp/app/v3/runOpsMigration/unblockRouteCatalog.ts @@ -65,6 +65,12 @@ export const UNBLOCK_ROUTES: readonly UnblockRoute[] = [ site: BATCH_SYSTEM, symbol: "#tryCompleteBatch", }, + { + id: "batch.expireBatch", + kind: "RUN", + site: BATCH_SYSTEM, + symbol: "expireBatch", + }, { id: "ttl.expireRun", kind: "RUN", diff --git a/apps/webapp/test/authorizationRateLimitMiddleware.test.ts b/apps/webapp/test/authorizationRateLimitMiddleware.test.ts index 847327b4b7a..b6076cef0de 100644 --- a/apps/webapp/test/authorizationRateLimitMiddleware.test.ts +++ b/apps/webapp/test/authorizationRateLimitMiddleware.test.ts @@ -25,7 +25,7 @@ describe.skipIf(process.env.GITHUB_ACTIONS)("authorizationRateLimitMiddleware", redisTest("should allow requests within the rate limit", async ({ redisOptions }) => { const rateLimitMiddleware = authorizationRateLimitMiddleware({ - redis: redisOptions, + redis: { ...redisOptions, tlsDisabled: true }, keyPrefix: "test", defaultLimiter: { type: "tokenBucket", @@ -56,7 +56,7 @@ describe.skipIf(process.env.GITHUB_ACTIONS)("authorizationRateLimitMiddleware", redisTest("should reject requests without an Authorization header", async ({ redisOptions }) => { const rateLimitMiddleware = authorizationRateLimitMiddleware({ - redis: redisOptions, + redis: { ...redisOptions, tlsDisabled: true }, keyPrefix: "test", defaultLimiter: { type: "tokenBucket", @@ -80,7 +80,7 @@ describe.skipIf(process.env.GITHUB_ACTIONS)("authorizationRateLimitMiddleware", redisTest("should reject requests that exceed the rate limit", async ({ redisOptions }) => { const rateLimitMiddleware = authorizationRateLimitMiddleware({ - redis: redisOptions, + redis: { ...redisOptions, tlsDisabled: true }, keyPrefix: "test", defaultLimiter: { type: "tokenBucket", @@ -108,7 +108,7 @@ describe.skipIf(process.env.GITHUB_ACTIONS)("authorizationRateLimitMiddleware", redisTest("should not apply rate limiting to whitelisted paths", async ({ redisOptions }) => { const rateLimitMiddleware = authorizationRateLimitMiddleware({ - redis: redisOptions, + redis: { ...redisOptions, tlsDisabled: true }, keyPrefix: "test", defaultLimiter: { type: "tokenBucket", @@ -138,7 +138,7 @@ describe.skipIf(process.env.GITHUB_ACTIONS)("authorizationRateLimitMiddleware", "should apply different rate limits based on limiterConfigOverride", async ({ redisOptions }) => { const rateLimitMiddleware = authorizationRateLimitMiddleware({ - redis: redisOptions, + redis: { ...redisOptions, tlsDisabled: true }, keyPrefix: "test", defaultLimiter: { type: "tokenBucket", @@ -188,7 +188,7 @@ describe.skipIf(process.env.GITHUB_ACTIONS)("authorizationRateLimitMiddleware", // 1. Test different rate limit configurations redisTest("should enforce fixed window rate limiting", async ({ redisOptions }) => { const rateLimitMiddleware = authorizationRateLimitMiddleware({ - redis: redisOptions, + redis: { ...redisOptions, tlsDisabled: true }, keyPrefix: "test-fixed", defaultLimiter: { type: "fixedWindow", @@ -224,7 +224,7 @@ describe.skipIf(process.env.GITHUB_ACTIONS)("authorizationRateLimitMiddleware", redisTest("should enforce sliding window rate limiting", async ({ redisOptions }) => { const rateLimitMiddleware = authorizationRateLimitMiddleware({ - redis: redisOptions, + redis: { ...redisOptions, tlsDisabled: true }, keyPrefix: "test-sliding", defaultLimiter: { type: "slidingWindow", @@ -268,7 +268,7 @@ describe.skipIf(process.env.GITHUB_ACTIONS)("authorizationRateLimitMiddleware", // 2. Test edge cases around rate limit calculations redisTest("should handle token refill correctly", async ({ redisOptions }) => { const rateLimitMiddleware = authorizationRateLimitMiddleware({ - redis: redisOptions, + redis: { ...redisOptions, tlsDisabled: true }, keyPrefix: "test-refill", defaultLimiter: { type: "tokenBucket", @@ -309,7 +309,7 @@ describe.skipIf(process.env.GITHUB_ACTIONS)("authorizationRateLimitMiddleware", redisTest("should handle near-zero remaining tokens correctly", async ({ redisOptions }) => { const rateLimitMiddleware = authorizationRateLimitMiddleware({ - redis: redisOptions, + redis: { ...redisOptions, tlsDisabled: true }, keyPrefix: "test-near-zero", defaultLimiter: { type: "tokenBucket", @@ -357,7 +357,7 @@ describe.skipIf(process.env.GITHUB_ACTIONS)("authorizationRateLimitMiddleware", redisTest("should use cached limiter configurations", async ({ redisOptions }) => { let configOverrideCalls = 0; const rateLimitMiddleware = authorizationRateLimitMiddleware({ - redis: redisOptions, + redis: { ...redisOptions, tlsDisabled: true }, keyPrefix: "test-cache", defaultLimiter: { type: "tokenBucket", diff --git a/apps/webapp/test/authorizationRateLimitMiddlewareBypass.test.ts b/apps/webapp/test/authorizationRateLimitMiddlewareBypass.test.ts new file mode 100644 index 00000000000..b9d6c175d27 --- /dev/null +++ b/apps/webapp/test/authorizationRateLimitMiddlewareBypass.test.ts @@ -0,0 +1,92 @@ +import { redisTest } from "@internal/testcontainers"; +import { beforeEach, describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 30_000 }); + +import type { Express } from "express"; +import express from "express"; +import request from "supertest"; +import { authorizationRateLimitMiddleware } from "../app/services/authorizationRateLimitMiddleware.server.js"; + +const exhaustedLimiter = { + type: "tokenBucket", + refillRate: 1, + interval: "1m", + maxTokens: 1, +} as const; + +describe("authorizationRateLimitMiddleware bypass", () => { + let app: Express; + + beforeEach(() => { + app = express(); + }); + + redisTest("lets a bypassed request through an exhausted limit", async ({ redisOptions }) => { + const rateLimitMiddleware = authorizationRateLimitMiddleware({ + redis: { ...redisOptions, tlsDisabled: true }, + keyPrefix: "test-bypass-allowed", + defaultLimiter: exhaustedLimiter, + pathMatchers: [/^\/api/], + bypass: async (req) => req.path === "/api/granted", + }); + + app.use(rateLimitMiddleware); + app.get("/api/granted", (req, res) => res.status(200).json({ message: "Granted" })); + app.get("/api/limited", (req, res) => res.status(200).json({ message: "Limited" })); + + await request(app).get("/api/limited").set("Authorization", "Bearer test-token"); + const limited = await request(app) + .get("/api/limited") + .set("Authorization", "Bearer test-token"); + expect(limited.status).toBe(429); + + const granted = await request(app) + .get("/api/granted") + .set("Authorization", "Bearer test-token"); + + expect(granted.status).toBe(200); + expect(granted.body).toEqual({ message: "Granted" }); + }); + + redisTest("falls back to the limiter when the bypass declines", async ({ redisOptions }) => { + const rateLimitMiddleware = authorizationRateLimitMiddleware({ + redis: { ...redisOptions, tlsDisabled: true }, + keyPrefix: "test-bypass-declined", + defaultLimiter: exhaustedLimiter, + pathMatchers: [/^\/api/], + bypass: async () => false, + }); + + app.use(rateLimitMiddleware); + app.get("/api/test", (req, res) => res.status(200).json({ message: "Success" })); + + await request(app).get("/api/test").set("Authorization", "Bearer declined"); + const response = await request(app).get("/api/test").set("Authorization", "Bearer declined"); + + expect(response.status).toBe(429); + }); + + redisTest("does not let the bypass skip authentication", async ({ redisOptions }) => { + let bypassCalled = false; + + const rateLimitMiddleware = authorizationRateLimitMiddleware({ + redis: { ...redisOptions, tlsDisabled: true }, + keyPrefix: "test-bypass-unauthenticated", + defaultLimiter: exhaustedLimiter, + pathMatchers: [/^\/api/], + bypass: async () => { + bypassCalled = true; + return true; + }, + }); + + app.use(rateLimitMiddleware); + app.get("/api/test", (req, res) => res.status(200).json({ message: "Success" })); + + const response = await request(app).get("/api/test"); + + expect(response.status).toBe(401); + expect(bypassCalled).toBe(false); + }); +}); diff --git a/apps/webapp/test/batchStreamGrants.test.ts b/apps/webapp/test/batchStreamGrants.test.ts new file mode 100644 index 00000000000..82fbd50aa88 --- /dev/null +++ b/apps/webapp/test/batchStreamGrants.test.ts @@ -0,0 +1,102 @@ +import { redisTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 30_000 }); + +import { BatchStreamGrants } from "../app/runEngine/concerns/batchStreamGrants.server.js"; + +describe("BatchStreamGrants", () => { + const ENV = "env_1"; + + redisTest("spends exactly the granted number of attempts", async ({ redisOptions }) => { + const grants = new BatchStreamGrants({ + redis: { ...redisOptions, tlsDisabled: true }, + attempts: 3, + ttlMs: 60_000, + }); + + try { + await grants.mint(ENV, "batch_spend"); + + expect(await grants.spend(ENV, "batch_spend")).toBe(true); + expect(await grants.spend(ENV, "batch_spend")).toBe(true); + expect(await grants.spend(ENV, "batch_spend")).toBe(true); + expect(await grants.spend(ENV, "batch_spend")).toBe(false); + expect(await grants.spend(ENV, "batch_spend")).toBe(false); + } finally { + await grants.quit(); + } + }); + + redisTest("declines a batch that was never granted", async ({ redisOptions }) => { + const grants = new BatchStreamGrants({ + redis: { ...redisOptions, tlsDisabled: true }, + attempts: 5, + ttlMs: 60_000, + }); + + try { + expect(await grants.spend(ENV, "batch_never_minted")).toBe(false); + } finally { + await grants.quit(); + } + }); + + redisTest("keeps grants isolated per batch", async ({ redisOptions }) => { + const grants = new BatchStreamGrants({ + redis: { ...redisOptions, tlsDisabled: true }, + attempts: 1, + ttlMs: 60_000, + }); + + try { + await grants.mint(ENV, "batch_a"); + await grants.mint(ENV, "batch_b"); + + expect(await grants.spend(ENV, "batch_a")).toBe(true); + expect(await grants.spend(ENV, "batch_a")).toBe(false); + expect(await grants.spend(ENV, "batch_b")).toBe(true); + } finally { + await grants.quit(); + } + }); + + redisTest("expires the grant so it cannot outlive the seal window", async ({ redisOptions }) => { + const grants = new BatchStreamGrants({ + redis: { ...redisOptions, tlsDisabled: true }, + attempts: 100_000, + ttlMs: 150, + }); + + try { + await grants.mint(ENV, "batch_expiring"); + expect(await grants.spend(ENV, "batch_expiring")).toBe(true); + + await vi.waitFor( + async () => { + expect(await grants.spend(ENV, "batch_expiring")).toBe(false); + }, + { timeout: 5_000, interval: 50 } + ); + } finally { + await grants.quit(); + } + }); + + redisTest("another environment cannot spend this batch's grant", async ({ redisOptions }) => { + const grants = new BatchStreamGrants({ + redis: { ...redisOptions, tlsDisabled: true }, + attempts: 1, + ttlMs: 60_000, + }); + + try { + await grants.mint(ENV, "batch_scoped"); + + expect(await grants.spend("env_intruder", "batch_scoped")).toBe(false); + expect(await grants.spend(ENV, "batch_scoped")).toBe(true); + } finally { + await grants.quit(); + } + }); +}); diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index b5131766559..07de80bed88 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -286,6 +286,9 @@ export class RunEngine { tryCompleteBatch: async ({ payload }) => { await this.batchSystem.performCompleteBatch({ batchId: payload.batchId }); }, + expireBatch: async ({ payload }) => { + await this.batchSystem.expireBatch({ batchId: payload.batchId }); + }, continueRunIfUnblocked: async ({ payload }) => { await this.waitpointSystem.continueRunIfUnblocked({ runId: payload.runId, @@ -1812,6 +1815,28 @@ export class RunEngine { return this.batchSystem.scheduleCompleteBatch({ batchId }); } + /** + * Terminally fail a batch whose phase 2 item stream never sealed it, completing the + * parent's batchTriggerAndWait waitpoint with an error so the parent resumes. + */ + async expireBatch({ batchId }: { batchId: string }): Promise<void> { + return this.batchSystem.expireBatch({ batchId }); + } + + /** + * Schedule the seal-timeout reaper. Only worth scheduling for a batch that blocks a + * parent, since a fire-and-forget batch has nothing to strand. + */ + async scheduleExpireBatch({ + batchId, + availableAt, + }: { + batchId: string; + availableAt: Date; + }): Promise<void> { + return this.batchSystem.scheduleExpireBatch({ batchId, availableAt }); + } + // ============================================================================ // BatchQueue methods (DRR-based batch processing) // ============================================================================ diff --git a/internal-packages/run-engine/src/engine/systems/batchSystem.test.ts b/internal-packages/run-engine/src/engine/systems/batchSystem.test.ts index 39a0f965e13..defd36f6147 100644 --- a/internal-packages/run-engine/src/engine/systems/batchSystem.test.ts +++ b/internal-packages/run-engine/src/engine/systems/batchSystem.test.ts @@ -88,6 +88,14 @@ class CountingPostgresRunStore extends PostgresRunStore { return super.updateBatchTaskRun(args, tx); } + override async updateManyBatchTaskRun( + args: Prisma.BatchTaskRunUpdateManyArgs, + tx?: any + ): Promise<Prisma.BatchPayload> { + this.batchUpdates++; + return super.updateManyBatchTaskRun(args, tx); + } + override async findWaitpoint<T extends Prisma.WaitpointFindFirstArgs>( args: any, client?: any diff --git a/internal-packages/run-engine/src/engine/systems/batchSystem.ts b/internal-packages/run-engine/src/engine/systems/batchSystem.ts index 7ca78703dd8..6b100c5f82c 100644 --- a/internal-packages/run-engine/src/engine/systems/batchSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/batchSystem.ts @@ -1,4 +1,5 @@ -import { startSpan } from "@internal/tracing"; +import { startSpan, type Counter } from "@internal/tracing"; +import type { TaskRunError } from "@trigger.dev/core/v3/schemas"; import { isFinalRunStatus } from "../statuses.js"; import type { SystemResources } from "./systems.js"; import type { WaitpointSystem } from "./waitpointSystem.js"; @@ -11,10 +12,15 @@ export type BatchSystemOptions = { export class BatchSystem { private readonly $: SystemResources; private readonly waitpointSystem: WaitpointSystem; + private readonly expirationsCounter: Counter; constructor(private readonly options: BatchSystemOptions) { this.$ = options.resources; this.waitpointSystem = options.waitpointSystem; + this.expirationsCounter = this.$.meter.createCounter("batch_system.expirations", { + description: "Seal-timeout reaper runs, by outcome", + unit: "batches", + }); } public async scheduleCompleteBatch({ batchId }: { batchId: string }): Promise<void> { @@ -31,6 +37,125 @@ export class BatchSystem { await this.#tryCompleteBatch({ batchId }); } + public async scheduleExpireBatch({ + batchId, + availableAt, + }: { + batchId: string; + availableAt: Date; + }): Promise<void> { + await this.$.worker.enqueue({ + id: `expireBatch:${batchId}`, + job: "expireBatch", + payload: { batchId }, + availableAt, + }); + } + + /** + * Terminally fail a batch whose phase 2 item stream never sealed it, completing the + * parent's batchTriggerAndWait waitpoint with an error so the parent resumes with a + * failure instead of hanging forever. + * + * A batch is only sealed once every item has streamed, but the parent is blocked on the + * batch's waitpoint from the moment the batch is created. Nothing else completes that + * waitpoint, so without this a stream that never finishes strands the parent permanently. + * + * Idempotent and race-safe: a batch that sealed in the meantime is left alone. + * + * Resumable too. The abort and the waitpoint completion are two writes, so a crash between + * them would otherwise leave the parent blocked with no way back: the retry would see a + * non-PENDING batch and bail. An already-ABORTED batch therefore falls through to complete + * its waitpoint again, which {@link WaitpointSystem.completeWaitpoint} treats as a no-op. + */ + public async expireBatch({ batchId }: { batchId: string }): Promise<void> { + return startSpan(this.$.tracer, "expireBatch", async (span) => { + span.setAttribute("batchId", batchId); + + const batch = await this.$.runStore.findBatchTaskRunById(batchId, undefined, this.$.prisma); + + if (!batch) { + this.$.logger.debug("expireBatch: batch doesn't exist", { batchId }); + this.expirationsCounter.add(1, { outcome: "missing" }); + return; + } + + if (batch.sealed || (batch.status !== "PENDING" && batch.status !== "ABORTED")) { + this.$.logger.debug("expireBatch: batch sealed or already finished, nothing to do", { + batchId, + status: batch.status, + sealed: batch.sealed, + }); + this.expirationsCounter.add(1, { outcome: "already_settled" }); + return; + } + + if (batch.status === "PENDING") { + const aborted = await this.$.runStore.updateManyBatchTaskRun( + { + where: { id: batchId, sealed: false, status: "PENDING" }, + data: { + status: "ABORTED", + completedAt: new Date(), + processingCompletedAt: new Date(), + }, + }, + this.$.prisma + ); + + if (aborted.count === 0) { + const current = await this.$.runStore.findBatchTaskRunById( + batchId, + undefined, + this.$.prisma + ); + + if (!current || current.sealed || current.status !== "ABORTED") { + this.$.logger.debug("expireBatch: lost the race to a seal, no-op", { batchId }); + this.expirationsCounter.add(1, { outcome: "lost_race" }); + return; + } + } + } + + const waitpoint = await this.$.runStore.findWaitpoint( + { + where: { completedByBatchId: batchId }, + }, + this.$.prisma + ); + + if (!waitpoint) { + this.$.logger.debug("expireBatch: no waitpoint, nothing was blocked on this batch", { + batchId, + }); + this.expirationsCounter.add(1, { outcome: "aborted_no_parent" }); + return; + } + + const error: TaskRunError = { + type: "STRING_ERROR", + raw: + `Batch ${batch.friendlyId} was never fully created: its ${batch.expectedCount} ` + + `items could not be streamed before it timed out, so the batch can never complete. ` + + `batchTriggerAndWait failed rather than waiting forever.`, + }; + + await this.waitpointSystem.completeWaitpoint({ + id: waitpoint.id, + output: { value: JSON.stringify(error), isError: true }, + }); + + this.expirationsCounter.add(1, { outcome: "aborted_parent_resumed" }); + + this.$.logger.warn("expireBatch: aborted an unsealed batch and resumed its parent", { + batchId, + waitpointId: waitpoint.id, + expectedCount: batch.expectedCount, + }); + }); + } + /** * Checks to see if all runs for a BatchTaskRun are completed, if they are then update the status. * This isn't used operationally, but it's used for the Batches dashboard page. @@ -89,21 +214,22 @@ export class BatchSystem { if (runs.every((r) => isFinalRunStatus(r.status))) { this.$.logger.debug("#tryCompleteBatch: All runs are completed", { batchId }); - await this.$.runStore.updateBatchTaskRun( + + const completed = await this.$.runStore.updateManyBatchTaskRun( { - where: { - id: batchId, - }, - data: { - status: "COMPLETED", - }, - select: { - id: true, - }, + where: { id: batchId, status: { notIn: ["ABORTED", "COMPLETED"] } }, + data: { status: "COMPLETED" }, }, this.$.prisma ); + if (completed.count === 0) { + this.$.logger.debug("#tryCompleteBatch: batch already reached a terminal status", { + batchId, + }); + return; + } + //get waitpoint (if there is one) const waitpoint = await this.$.runStore.findWaitpoint( { diff --git a/internal-packages/run-engine/src/engine/tests/batchTwoPhase.test.ts b/internal-packages/run-engine/src/engine/tests/batchTwoPhase.test.ts index 8471c07844b..106a4c6b73b 100644 --- a/internal-packages/run-engine/src/engine/tests/batchTwoPhase.test.ts +++ b/internal-packages/run-engine/src/engine/tests/batchTwoPhase.test.ts @@ -625,4 +625,324 @@ describe("RunEngine 2-Phase Batch API", () => { } } ); + + describe("seal-timeout reaper", () => { + type TestPrisma = Parameters<typeof setupAuthenticatedEnvironment>[0]; + type TestEnvironment = Awaited<ReturnType<typeof setupAuthenticatedEnvironment>>; + type TestRedisOptions = ConstructorParameters<typeof RunEngine>[0]["runLock"]["redis"]; + + function createEngine(prisma: TestPrisma, redisOptions: TestRedisOptions) { + const engine = new RunEngine({ + prisma, + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 20 }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + batchQueue: { + redis: redisOptions, + consumerCount: 2, + consumerIntervalMs: 50, + drr: { quantum: 10, maxDeficit: 100 }, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + engine.setBatchProcessItemCallback(async () => ({ + success: true, + runId: "should-not-be-called", + })); + engine.setBatchCompletionCallback(async () => {}); + + return engine; + } + + /** + * Phase 1 without phase 2: a created, unsealed batch with its parent blocked on the + * batch waitpoint. This is the exact state a stream that never completes leaves behind. + */ + async function setupUnsealedBatchBlockingParent( + engine: RunEngine, + prisma: TestPrisma, + authenticatedEnvironment: TestEnvironment, + expectedCount = 2 + ) { + const parentTask = "parent-task"; + await setupBackgroundWorker(engine, authenticatedEnvironment, [parentTask]); + + const { id: batchId, friendlyId: batchFriendlyId } = BatchId.generate(); + await prisma.batchTaskRun.create({ + data: { + id: batchId, + friendlyId: batchFriendlyId, + runtimeEnvironmentId: authenticatedEnvironment.id, + status: "PENDING", + runCount: expectedCount, + expectedCount, + sealed: false, + batchVersion: "runengine:v2", + }, + }); + + const parentRun = await engine.trigger( + { + friendlyId: generateFriendlyId("run"), + environment: authenticatedEnvironment, + taskIdentifier: parentTask, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t_parent", + spanId: "s_parent", + workerQueue: "main", + queue: `task/${parentTask}`, + isTest: false, + tags: [], + }, + prisma + ); + + await vi.waitFor( + async () => { + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "test_12345", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + }, + { timeout: 15_000, interval: 100 } + ); + + const initialExecutionData = await engine.getRunExecutionData({ runId: parentRun.id }); + assertNonNullable(initialExecutionData); + await engine.startRunAttempt({ + runId: parentRun.id, + snapshotId: initialExecutionData.snapshot.id, + }); + + await engine.blockRunWithCreatedBatch({ + runId: parentRun.id, + batchId, + environmentId: authenticatedEnvironment.id, + projectId: authenticatedEnvironment.projectId, + organizationId: authenticatedEnvironment.organizationId, + }); + + await engine.initializeBatch({ + batchId, + friendlyId: batchFriendlyId, + environmentId: authenticatedEnvironment.id, + environmentType: authenticatedEnvironment.type, + organizationId: authenticatedEnvironment.organizationId, + projectId: authenticatedEnvironment.projectId, + runCount: expectedCount, + parentRunId: parentRun.id, + resumeParentOnCompletion: true, + }); + + const afterBlocked = await engine.getRunExecutionData({ runId: parentRun.id }); + assertNonNullable(afterBlocked); + expect(afterBlocked.snapshot.executionStatus).toBe("EXECUTING_WITH_WAITPOINTS"); + + return { batchId, batchFriendlyId, parentRun }; + } + + containerTest( + "aborts an unsealed batch and resumes the parent with an error", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const engine = createEngine(prisma, redisOptions); + + try { + const { batchId, parentRun } = await setupUnsealedBatchBlockingParent( + engine, + prisma, + authenticatedEnvironment + ); + + await engine.expireBatch({ batchId }); + + const abortedBatch = await prisma.batchTaskRun.findFirst({ where: { id: batchId } }); + expect(abortedBatch?.status).toBe("ABORTED"); + expect(abortedBatch?.completedAt).not.toBeNull(); + + const waitpoint = await prisma.waitpoint.findFirst({ + where: { completedByBatchId: batchId }, + }); + assertNonNullable(waitpoint); + expect(waitpoint.status).toBe("COMPLETED"); + expect(waitpoint.outputIsError).toBe(true); + + await vi.waitFor( + async () => { + const waitpoints = await prisma.taskRunWaitpoint.findMany({ + where: { taskRunId: parentRun.id }, + }); + expect(waitpoints.length).toBe(0); + }, + { timeout: 15_000 } + ); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "leaves a batch alone when the stream sealed it first", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const engine = createEngine(prisma, redisOptions); + + try { + const { batchId } = await setupUnsealedBatchBlockingParent( + engine, + prisma, + authenticatedEnvironment + ); + + await prisma.batchTaskRun.update({ + where: { id: batchId }, + data: { sealed: true, sealedAt: new Date() }, + }); + + await engine.expireBatch({ batchId }); + + const batch = await prisma.batchTaskRun.findFirst({ where: { id: batchId } }); + expect(batch?.status).toBe("PENDING"); + + const waitpoint = await prisma.waitpoint.findFirst({ + where: { completedByBatchId: batchId }, + }); + assertNonNullable(waitpoint); + expect(waitpoint.status).toBe("PENDING"); + } finally { + await engine.quit(); + } + } + ); + + containerTest("is idempotent when it runs twice", async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const engine = createEngine(prisma, redisOptions); + + try { + const { batchId } = await setupUnsealedBatchBlockingParent( + engine, + prisma, + authenticatedEnvironment + ); + + await engine.expireBatch({ batchId }); + const afterFirst = await prisma.batchTaskRun.findFirst({ where: { id: batchId } }); + + await engine.expireBatch({ batchId }); + const afterSecond = await prisma.batchTaskRun.findFirst({ where: { id: batchId } }); + + expect(afterSecond?.status).toBe("ABORTED"); + expect(afterSecond?.completedAt?.getTime()).toBe(afterFirst?.completedAt?.getTime()); + + const waitpoints = await prisma.waitpoint.findMany({ + where: { completedByBatchId: batchId }, + }); + expect(waitpoints.length).toBe(1); + expect(waitpoints[0]?.status).toBe("COMPLETED"); + } finally { + await engine.quit(); + } + }); + + containerTest( + "resumes the parent when a previous run aborted the batch but died before the waitpoint", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const engine = createEngine(prisma, redisOptions); + + try { + const { batchId, parentRun } = await setupUnsealedBatchBlockingParent( + engine, + prisma, + authenticatedEnvironment + ); + + await prisma.batchTaskRun.update({ + where: { id: batchId }, + data: { + status: "ABORTED", + completedAt: new Date(), + processingCompletedAt: new Date(), + }, + }); + + const beforeRetry = await prisma.waitpoint.findFirst({ + where: { completedByBatchId: batchId }, + }); + assertNonNullable(beforeRetry); + expect(beforeRetry.status).toBe("PENDING"); + + await engine.expireBatch({ batchId }); + + const waitpoint = await prisma.waitpoint.findFirst({ + where: { completedByBatchId: batchId }, + }); + assertNonNullable(waitpoint); + expect(waitpoint.status).toBe("COMPLETED"); + expect(waitpoint.outputIsError).toBe(true); + + await vi.waitFor( + async () => { + const waitpoints = await prisma.taskRunWaitpoint.findMany({ + where: { taskRunId: parentRun.id }, + }); + expect(waitpoints.length).toBe(0); + }, + { timeout: 15_000 } + ); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "aborts a fire-and-forget batch without a waitpoint to resolve", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const engine = createEngine(prisma, redisOptions); + + try { + const { id: batchId, friendlyId } = BatchId.generate(); + await prisma.batchTaskRun.create({ + data: { + id: batchId, + friendlyId, + runtimeEnvironmentId: authenticatedEnvironment.id, + status: "PENDING", + runCount: 1, + expectedCount: 1, + sealed: false, + batchVersion: "runengine:v2", + }, + }); + + await engine.expireBatch({ batchId }); + + const batch = await prisma.batchTaskRun.findFirst({ where: { id: batchId } }); + expect(batch?.status).toBe("ABORTED"); + } finally { + await engine.quit(); + } + } + ); + }); }); diff --git a/internal-packages/run-engine/src/engine/workerCatalog.ts b/internal-packages/run-engine/src/engine/workerCatalog.ts index de54c1ece00..8f49d7d6068 100644 --- a/internal-packages/run-engine/src/engine/workerCatalog.ts +++ b/internal-packages/run-engine/src/engine/workerCatalog.ts @@ -58,6 +58,12 @@ export const workerCatalog = { }), visibilityTimeoutMs: 30_000, }, + expireBatch: { + schema: z.object({ + batchId: z.string(), + }), + visibilityTimeoutMs: 30_000, + }, continueRunIfUnblocked: { schema: z.object({ runId: z.string(), From a91c08c73171276b2e8698754995511a8d41ad3c Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:57:45 +0100 Subject: [PATCH 47/67] fix(core): retry run start-attempt on transient connection errors (#4441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What `startRunAttempt` — the run controller's first call when a run starts — had no retry on transient connection errors. A brief connection blip on that call would abandon the start and send the run back through the queue, delaying its first attempt. This adds a jittered backoff retry, matching the existing `continueRunExecution` path with a shorter budget, so a transient blip is ridden out in place instead of bouncing the run. ## Why a shorter budget The continue path retries generously. Start-attempt keeps a tighter budget (6 attempts, ~25-40s jittered) so it rides out a transient blip but never keeps retrying past the point the run would already have been requeued. ## Safety Retrying is safe: start-attempt is guarded server-side by the snapshot id — a retry after a start has already committed is rejected, so it can never double-start an attempt. A pure connection error (the common case) never reached the server. ## Scope One retry-options object on `startRunAttempt`; no other behavior change. Warm starts share this path and get the same resilience. --- .changeset/retry-run-start-on-connection-error.md | 5 +++++ packages/core/src/v3/runEngineWorker/workload/http.ts | 9 +++++++++ 2 files changed, 14 insertions(+) create mode 100644 .changeset/retry-run-start-on-connection-error.md diff --git a/.changeset/retry-run-start-on-connection-error.md b/.changeset/retry-run-start-on-connection-error.md new file mode 100644 index 00000000000..3624ee1b7c5 --- /dev/null +++ b/.changeset/retry-run-start-on-connection-error.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Transient connection errors when a run starts are now retried for longer, so a brief connectivity blip no longer sends the run back through the queue and delays its first attempt. diff --git a/packages/core/src/v3/runEngineWorker/workload/http.ts b/packages/core/src/v3/runEngineWorker/workload/http.ts index 43328b7244c..02e5d80b911 100644 --- a/packages/core/src/v3/runEngineWorker/workload/http.ts +++ b/packages/core/src/v3/runEngineWorker/workload/http.ts @@ -165,6 +165,15 @@ export class WorkloadHttpClient { ...this.defaultHeaders(), }, body: JSON.stringify(body), + }, + { + retry: { + minTimeoutInMs: 1000, + maxTimeoutInMs: 10_000, + maxAttempts: 6, + factor: 2, + randomize: true, + }, } ); } From f10bc23785e569e5d917318cf2033aabdbe96a0b Mon Sep 17 00:00:00 2001 From: Eric Allam <eric@trigger.dev> Date: Fri, 31 Jul 2026 16:12:07 +0100 Subject: [PATCH 48/67] perf(run-engine,run-store): one execution snapshot per triggered run (#4419) A non-delayed run used to get two execution snapshots the moment it was triggered: `RUN_CREATED` nested in the run-create transaction, immediately followed by `QUEUED` from its own `BEGIN`/`INSERT`/`COMMIT`. It now gets a single `QUEUED` snapshot written inside the create, and the trigger path only publishes to the queue. One fewer row per run on `TaskRunExecutionSnapshot`, and one fewer round trip on the trigger hot path. `EnqueueSystem` gains a `publishRun` seam that enqueues without writing a snapshot. Every re-enqueue path (waitpoint resume, checkpoint restore, delayed enqueue, pending version, retry requeue) still calls `enqueueRun` and writes its own `QUEUED`, so only the first enqueue changes. The `QUEUED` snapshot still commits before the queue message, so a dequeue sees a dequeueable status exactly as before. Two things for reviewers. Nesting the write skips `createExecutionSnapshot`, which is what emits `executionSnapshotCreated` and therefore the run timeline's `[engine] QUEUED` entry, so the trigger path now emits it directly, the same way the dequeue and attempt-start paths already do for their nested creates. And `RUN_CREATED` is still written when a dequeued run has no background worker yet, so the status and both `statuses.ts` helpers stay live and existing rows keep reading correctly. Delayed runs are untouched: `DELAYED` then `QUEUED` are two genuinely different moments and stay two snapshots. Rollback is a revert. Create-and-enqueue happen in one request in one process, so no in-flight run needs both code paths to agree during a rollout. One note for whoever debugs this path later. The `QUEUED` snapshot now commits before the queue publish, so a failed publish leaves the run recorded as `QUEUED` with no queue message. That state was already reachable, since the publish was never part of the snapshot transaction, but it used to be recorded as `RUN_CREATED`, which was distinctive because it never otherwise persisted. `QUEUED` with no message is indistinguishable from a run waiting on a concurrency slot, so trigger-time publish failure is now one more cause of an apparently stuck queued run. --- .../collapse-trigger-queued-snapshot.md | 6 + .../run-engine/src/engine/consts.ts | 10 + .../run-engine/src/engine/index.ts | 34 +- .../src/engine/systems/enqueueSystem.test.ts | 32 +- .../src/engine/systems/enqueueSystem.ts | 99 +++-- .../systems/executionSnapshotSystem.test.ts | 8 + .../engine/tests/getSnapshotsSince.test.ts | 11 +- .../engine/tests/triggerCreateRouting.test.ts | 5 +- .../tests/triggerSnapshotCollapse.test.ts | 370 ++++++++++++++++++ .../run-store/src/PostgresRunStore.ts | 1 + internal-packages/run-store/src/types.ts | 1 + 11 files changed, 522 insertions(+), 55 deletions(-) create mode 100644 .server-changes/collapse-trigger-queued-snapshot.md create mode 100644 internal-packages/run-engine/src/engine/tests/triggerSnapshotCollapse.test.ts diff --git a/.server-changes/collapse-trigger-queued-snapshot.md b/.server-changes/collapse-trigger-queued-snapshot.md new file mode 100644 index 00000000000..dffc9671e6b --- /dev/null +++ b/.server-changes/collapse-trigger-queued-snapshot.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Triggering a task now does one fewer database write, so runs reach the queue slightly faster. diff --git a/internal-packages/run-engine/src/engine/consts.ts b/internal-packages/run-engine/src/engine/consts.ts index 6ea6f54c38e..3f0bb59eb4a 100644 --- a/internal-packages/run-engine/src/engine/consts.ts +++ b/internal-packages/run-engine/src/engine/consts.ts @@ -1 +1,11 @@ export const MAX_TASK_RUN_ATTEMPTS = 250; + +/** + * The status and description a run's default entry into the queue is written with. Shared because + * the trigger path writes this snapshot nested in the run-create transaction and then emits its own + * `executionSnapshotCreated`, while every re-enqueue writes it through `enqueueRun`. Three places + * have to agree, or the persisted row and the run timeline's `[engine]` entry drift apart. + * Re-enqueues that describe why they requeued pass their own description instead. + */ +export const QUEUED_SNAPSHOT_STATUS = "QUEUED" as const; +export const QUEUED_SNAPSHOT_DESCRIPTION = "Run was QUEUED"; diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 07de80bed88..b88f4f276e4 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -23,6 +23,7 @@ import { } from "@trigger.dev/core/v3"; import type { TaskRunError } from "@trigger.dev/core/v3/schemas"; import { + generateInternalId, parseNaturalLanguageDurationInMs, RunId, WaitpointId, @@ -53,6 +54,7 @@ import { RunQueue } from "../run-queue/index.js"; import { RunQueueFullKeyProducer } from "../run-queue/keyProducer.js"; import type { AuthenticatedEnvironment, MinimalAuthenticatedEnvironment } from "../shared/index.js"; import { BillingCache } from "./billingCache.js"; +import { QUEUED_SNAPSHOT_DESCRIPTION, QUEUED_SNAPSHOT_STATUS } from "./consts.js"; import { ExecutionSnapshotNotFoundError, NotImplementedError, @@ -950,6 +952,7 @@ export class RunEngine { let taskRun: TaskRun & { associatedWaitpoint: Waitpoint | null }; const taskRunId = RunId.fromFriendlyId(friendlyId); + const initialSnapshotId = generateInternalId(); // App-level replacement for the dropped TaskRun env/project Cascade FKs. await this.controlPlaneResolver.assertEnvExists(environment.id); @@ -1035,9 +1038,10 @@ export class RunEngine { annotations, }, snapshot: { + id: initialSnapshotId, engine: "V2", - executionStatus: delayUntil ? "DELAYED" : "RUN_CREATED", - description: delayUntil ? "Run is delayed" : "Run was created", + executionStatus: delayUntil ? "DELAYED" : QUEUED_SNAPSHOT_STATUS, + description: delayUntil ? "Run is delayed" : QUEUED_SNAPSHOT_DESCRIPTION, runStatus: status, environmentId: environment.id, environmentType: environment.type, @@ -1164,13 +1168,29 @@ export class RunEngine { await this.ttlSystem.scheduleExpireRun({ runId: taskRun.id, ttl: taskRun.ttl }); } - await this.enqueueSystem.enqueueRun({ + this.eventBus.emit("executionSnapshotCreated", { + time: new Date(), + run: { + id: taskRun.id, + }, + snapshot: { + id: initialSnapshotId, + executionStatus: QUEUED_SNAPSHOT_STATUS, + description: QUEUED_SNAPSHOT_DESCRIPTION, + runStatus: taskRun.status, + attemptNumber: taskRun.attemptNumber ?? null, + checkpointId: null, + workerId: workerId ?? null, + runnerId: runnerId ?? null, + isValid: true, + error: null, + completedWaitpointIds: [], + }, + }); + + await this.enqueueSystem.publishRun({ run: taskRun, env: environment, - workerId, - runnerId, - tx: prisma, - skipRunLock: true, includeTtl: true, anchorEligibilityAtQueuePosition: true, enableFastPath, diff --git a/internal-packages/run-engine/src/engine/systems/enqueueSystem.test.ts b/internal-packages/run-engine/src/engine/systems/enqueueSystem.test.ts index a5730b306f2..8903129fc85 100644 --- a/internal-packages/run-engine/src/engine/systems/enqueueSystem.test.ts +++ b/internal-packages/run-engine/src/engine/systems/enqueueSystem.test.ts @@ -44,27 +44,37 @@ function createEngineOptions(redisOptions: any, prisma: any, store?: PostgresRun } /** - * A real PostgresRunStore subclass that counts the snapshot create method that enqueueRun's - * snapshot write routes through (via executionSnapshotSystem.createExecutionSnapshot). super.* - * runs the genuine store implementation, so the routing is observed over real containers without - * ever mocking prisma or the store. + * A real PostgresRunStore subclass that counts the store methods a run's QUEUED snapshot can be + * written through: nested in `createRun` on the trigger path, or standalone via + * `createExecutionSnapshot` on every re-enqueue. super.* runs the genuine store implementation, so + * the routing is observed over real containers without ever mocking prisma or the store. */ class CountingPostgresRunStore extends PostgresRunStore { - public snapshotCreates = 0; + /** Snapshots written nested in a run create: the trigger path's QUEUED write. */ + public nestedSnapshotCreates = 0; + /** Snapshots written standalone through `createExecutionSnapshot`: every re-enqueue. */ + public standaloneSnapshotCreates = 0; override async createExecutionSnapshot( input: any, tx?: any ): ReturnType<PostgresRunStore["createExecutionSnapshot"]> { - this.snapshotCreates++; + this.standaloneSnapshotCreates++; return super.createExecutionSnapshot(input, tx); } + + override async createRun( + params: Parameters<PostgresRunStore["createRun"]>[0], + tx?: any + ): ReturnType<PostgresRunStore["createRun"]> { + this.nestedSnapshotCreates++; + return super.createRun(params, tx); + } } describe("RunEngine enqueueRun store routing", () => { - // The QUEUED snapshot written while enqueuing a run routes through the injected store. containerTest( - "enqueueRun snapshot routes through the store", + "the QUEUED snapshot routes through the store as a single nested write", async ({ prisma, redisOptions }) => { const countingStore = new CountingPostgresRunStore({ prisma, readOnlyPrisma: prisma }); const engine = new RunEngine(createEngineOptions(redisOptions, prisma, countingStore)); @@ -74,7 +84,8 @@ describe("RunEngine enqueueRun store routing", () => { const taskIdentifier = "test-task"; await setupBackgroundWorker(engine, environment, taskIdentifier); - const before = countingStore.snapshotCreates; + const nestedBefore = countingStore.nestedSnapshotCreates; + const standaloneBefore = countingStore.standaloneSnapshotCreates; const run = await engine.trigger( { @@ -96,7 +107,8 @@ describe("RunEngine enqueueRun store routing", () => { prisma ); - expect(countingStore.snapshotCreates).toBeGreaterThan(before); + expect(countingStore.nestedSnapshotCreates).toBe(nestedBefore + 1); + expect(countingStore.standaloneSnapshotCreates).toBe(standaloneBefore); const latest = await getLatestExecutionSnapshot(prisma, run.id); assertNonNullable(latest); diff --git a/internal-packages/run-engine/src/engine/systems/enqueueSystem.ts b/internal-packages/run-engine/src/engine/systems/enqueueSystem.ts index 2b5aac6d376..38c681c511e 100644 --- a/internal-packages/run-engine/src/engine/systems/enqueueSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/enqueueSystem.ts @@ -7,6 +7,7 @@ import type { import type { RunStore } from "@internal/run-store"; import { parseNaturalLanguageDuration } from "@trigger.dev/core/v3/isomorphic"; import type { MinimalAuthenticatedEnvironment } from "../../shared/index.js"; +import { QUEUED_SNAPSHOT_DESCRIPTION, QUEUED_SNAPSHOT_STATUS } from "../consts.js"; import type { ExecutionSnapshotSystem } from "./executionSnapshotSystem.js"; import type { SystemResources } from "./systems.js"; @@ -95,8 +96,8 @@ export class EnqueueSystem { { run: run, snapshot: { - executionStatus: snapshot?.status ?? "QUEUED", - description: snapshot?.description ?? "Run was QUEUED", + executionStatus: snapshot?.status ?? QUEUED_SNAPSHOT_STATUS, + description: snapshot?.description ?? QUEUED_SNAPSHOT_DESCRIPTION, metadata: snapshot?.metadata ?? undefined, }, previousSnapshotId, @@ -113,42 +114,74 @@ export class EnqueueSystem { store ); - // Force development runs to use the environment id as the worker queue. - const workerQueue = env.type === "DEVELOPMENT" ? env.id : run.workerQueue; - - const queuePositionMs = (run.queueTimestamp ?? run.createdAt).getTime(); - const timestamp = queuePositionMs - run.priorityMs; - const eligibleAtMs = anchorEligibilityAtQueuePosition ? queuePositionMs : Date.now(); - - let ttlExpiresAt: number | undefined; - if (includeTtl && run.ttl) { - const expireAt = parseNaturalLanguageDuration(run.ttl); - if (expireAt) { - ttlExpiresAt = expireAt.getTime(); - } - } - - await this.$.runQueue.enqueueMessage({ + await this.publishRun({ + run, env, - workerQueue, + includeTtl, + anchorEligibilityAtQueuePosition, enableFastPath, - message: { - runId: run.id, - taskIdentifier: run.taskIdentifier, - orgId: env.organization.id, - projectId: env.project.id, - environmentId: env.id, - environmentType: env.type, - queue: run.queue, - concurrencyKey: run.concurrencyKey ?? undefined, - timestamp, - eligibleAtMs, - attempt: 0, - ttlExpiresAt, - }, }); return newSnapshot; }); } + + /** + * Publishes the run to the RunQueue without writing an execution snapshot. Callers that already + * hold a `QUEUED` snapshot (the trigger path writes one inside the run-create transaction) use + * this so the run does not pay for a second snapshot write. + */ + public async publishRun({ + run, + env, + includeTtl = false, + anchorEligibilityAtQueuePosition = false, + enableFastPath = false, + }: { + run: TaskRun; + env: MinimalAuthenticatedEnvironment; + /** See `enqueueRun`. */ + includeTtl?: boolean; + /** See `enqueueRun`. */ + anchorEligibilityAtQueuePosition?: boolean; + /** When true, allow the queue to push directly to worker queue if concurrency is available. */ + enableFastPath?: boolean; + }) { + // Force development runs to use the environment id as the worker queue. + const workerQueue = env.type === "DEVELOPMENT" ? env.id : run.workerQueue; + + const queuePositionMs = (run.queueTimestamp ?? run.createdAt).getTime(); + const timestamp = queuePositionMs - run.priorityMs; + const eligibleAtMs = anchorEligibilityAtQueuePosition ? queuePositionMs : Date.now(); + + // Include TTL only when explicitly requested (first enqueue from trigger). + // Re-enqueues (waitpoint, checkpoint, delayed, pending version) must not add TTL. + let ttlExpiresAt: number | undefined; + if (includeTtl && run.ttl) { + const expireAt = parseNaturalLanguageDuration(run.ttl); + if (expireAt) { + ttlExpiresAt = expireAt.getTime(); + } + } + + await this.$.runQueue.enqueueMessage({ + env, + workerQueue, + enableFastPath, + message: { + runId: run.id, + taskIdentifier: run.taskIdentifier, + orgId: env.organization.id, + projectId: env.project.id, + environmentId: env.id, + environmentType: env.type, + queue: run.queue, + concurrencyKey: run.concurrencyKey ?? undefined, + timestamp, + eligibleAtMs, + attempt: 0, + ttlExpiresAt, + }, + }); + } } diff --git a/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.test.ts b/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.test.ts index aeb361019e6..61d17a6b8b2 100644 --- a/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.test.ts +++ b/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.test.ts @@ -65,6 +65,14 @@ class CountingPostgresRunStore extends PostgresRunStore { return super.createExecutionSnapshot(input, tx); } + override async createRun( + params: Parameters<PostgresRunStore["createRun"]>[0], + tx?: any + ): ReturnType<PostgresRunStore["createRun"]> { + this.creates++; + return super.createRun(params, tx); + } + override async findLatestExecutionSnapshot( runId: string, client?: any diff --git a/internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts b/internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts index 61ff3bff75f..4cf74753e06 100644 --- a/internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts +++ b/internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts @@ -7,7 +7,7 @@ import type { PrismaClient } from "@trigger.dev/database"; import { RunEngine } from "../index.js"; import { getExecutionSnapshotsSince } from "../systems/executionSnapshotSystem.js"; import { copySnapshotsToReplica, createTestMetricsMeter } from "./helpers/replicaTestHelpers.js"; -import { setupTestScenario } from "./helpers/snapshotTestHelpers.js"; +import { createTestSnapshot, setupTestScenario } from "./helpers/snapshotTestHelpers.js"; import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; vi.setConfig({ testTimeout: 120_000 }); @@ -1154,6 +1154,15 @@ describe("RunEngine getSnapshotsSince", () => { workerQueue: "main", }); + await createTestSnapshot(prisma, { + runId: run.id, + status: "EXECUTING", + environmentId: authenticatedEnvironment.id, + environmentType: authenticatedEnvironment.type, + projectId: authenticatedEnvironment.project.id, + organizationId: authenticatedEnvironment.organization.id, + }); + const allSnapshots = await prisma.taskRunExecutionSnapshot.findMany({ where: { runId: run.id, isValid: true }, orderBy: { createdAt: "asc" }, diff --git a/internal-packages/run-engine/src/engine/tests/triggerCreateRouting.test.ts b/internal-packages/run-engine/src/engine/tests/triggerCreateRouting.test.ts index 34353cb44e0..af213dc7a71 100644 --- a/internal-packages/run-engine/src/engine/tests/triggerCreateRouting.test.ts +++ b/internal-packages/run-engine/src/engine/tests/triggerCreateRouting.test.ts @@ -137,9 +137,6 @@ const cancelledSnapshot = (friendlyId: string, environment: any) => ({ }); describe("RunEngine trigger/create routing", () => { - // trigger create routes through runStore.createRun with the structured - // DTO, and the persisted run + its nested first RUN_CREATED snapshot land via - // the single create call. containerTest( "trigger routes createRun and lands run + first snapshot", async ({ prisma, redisOptions }) => { @@ -169,7 +166,7 @@ describe("RunEngine trigger/create routing", () => { orderBy: { createdAt: "asc" }, }); expect(snapshot).not.toBeNull(); - expect(snapshot!.executionStatus).toBe("RUN_CREATED"); + expect(snapshot!.executionStatus).toBe("QUEUED"); } finally { await engine.quit(); } diff --git a/internal-packages/run-engine/src/engine/tests/triggerSnapshotCollapse.test.ts b/internal-packages/run-engine/src/engine/tests/triggerSnapshotCollapse.test.ts new file mode 100644 index 00000000000..bdb49419c08 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/triggerSnapshotCollapse.test.ts @@ -0,0 +1,370 @@ +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import type { PrismaClient, TaskRunExecutionStatus } from "@trigger.dev/database"; +import { setTimeout } from "node:timers/promises"; +import { expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; + +vi.setConfig({ testTimeout: 60_000 }); + +async function snapshotStatuses( + prisma: PrismaClient, + runId: string +): Promise<TaskRunExecutionStatus[]> { + const snapshots = await prisma.taskRunExecutionSnapshot.findMany({ + where: { runId }, + orderBy: { createdAt: "asc" }, + select: { executionStatus: true }, + }); + + return snapshots.map((snapshot) => snapshot.executionStatus); +} + +describe("RunEngine trigger() execution snapshots", () => { + containerTest( + "a non-delayed run is created with a single QUEUED snapshot", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_1234", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t_collapse_1", + spanId: "s_collapse_1", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + expect(await snapshotStatuses(prisma, run.id)).toEqual(["QUEUED"]); + + const queueLength = await engine.runQueue.lengthOfQueue( + authenticatedEnvironment, + run.queue + ); + expect(queueLength).toBe(1); + + const executionData = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executionData); + expect(executionData.snapshot.executionStatus).toBe("QUEUED"); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "the QUEUED snapshot event is stamped at write time, not at an overridden run createdAt", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + const stampedTimes: Date[] = []; + engine.eventBus.on("executionSnapshotCreated", ({ time }) => { + stampedTimes.push(time); + }); + + const backdatedCreatedAt = new Date(Date.now() - 60 * 60 * 1000); + const triggeredAt = Date.now(); + + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_1236", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t_collapse_3", + spanId: "s_collapse_3", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + createdAt: backdatedCreatedAt, + }, + prisma + ); + + const storedRun = await prisma.taskRun.findUnique({ where: { id: run.id } }); + expect(storedRun?.createdAt.getTime()).toBe(backdatedCreatedAt.getTime()); + + expect(stampedTimes.length).toBe(1); + expect(stampedTimes[0].getTime()).toBeGreaterThanOrEqual(triggeredAt); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "a delayed run keeps DELAYED and QUEUED as separate snapshots", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_1235", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t_collapse_2", + spanId: "s_collapse_2", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + delayUntil: new Date(Date.now() + 500), + }, + prisma + ); + + expect(await snapshotStatuses(prisma, run.id)).toEqual(["DELAYED"]); + + await setTimeout(1_500); + + expect(await snapshotStatuses(prisma, run.id)).toEqual(["DELAYED", "QUEUED"]); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "a resumed run still writes its own QUEUED snapshot", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + const run = await engine.trigger( + { + number: 1, + friendlyId: "run_1237", + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t_collapse_4", + spanId: "s_collapse_4", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + await setTimeout(500); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "test_collapse_4", + workerQueue: "main", + }); + assertNonNullable(dequeued[0]); + + await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + + const waitpointResult = await engine.createManualWaitpoint({ + environmentId: authenticatedEnvironment.id, + projectId: authenticatedEnvironment.projectId, + }); + + const blockedResult = await engine.blockRunWithWaitpoint({ + runId: run.id, + waitpoints: waitpointResult.waitpoint.id, + projectId: authenticatedEnvironment.projectId, + organizationId: authenticatedEnvironment.organizationId, + }); + + const checkpointResult = await engine.createCheckpoint({ + runId: run.id, + snapshotId: blockedResult.id, + checkpoint: { + type: "DOCKER", + reason: "TEST_CHECKPOINT", + location: "test-location", + imageRef: "test-image-ref", + }, + }); + expect(checkpointResult.ok).toBe(true); + + await engine.completeWaitpoint({ id: waitpointResult.waitpoint.id }); + await setTimeout(500); + + expect(await snapshotStatuses(prisma, run.id)).toEqual([ + "QUEUED", + "PENDING_EXECUTING", + "EXECUTING", + "EXECUTING_WITH_WAITPOINTS", + "SUSPENDED", + "QUEUED", + ]); + } finally { + await engine.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index 1b0d2f89cee..676e345ad5a 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -680,6 +680,7 @@ export class PostgresRunStore implements RunStore { const client = tx ?? this.prisma; const snapshotCreate = { + id: params.snapshot.id, engine: params.snapshot.engine, executionStatus: params.snapshot.executionStatus, description: params.snapshot.description, diff --git a/internal-packages/run-store/src/types.ts b/internal-packages/run-store/src/types.ts index 374a550e419..ac7cc35e3c8 100644 --- a/internal-packages/run-store/src/types.ts +++ b/internal-packages/run-store/src/types.ts @@ -29,6 +29,7 @@ export type IdempotencyKeyRunMatch = { }; export type CreateRunSnapshotInput = { + id?: string; engine: "V2"; executionStatus: TaskRunExecutionStatus; description: string; From b42e5c37716f0238efaf190a8cb47c8319b4c755 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Fri, 31 Jul 2026 19:45:37 +0100 Subject: [PATCH 49/67] fix(supervisor): count pods from a limit=1 list instead of an aggregate metric (#4442) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pod-count backpressure source read `apiserver_storage_objects{resource="pods"}` from an apiserver `/metrics` scrape. That gauge is a periodically-refreshed cached count, and it is served by whichever apiserver replica the scrape lands on — replicas disagree with each other at the same instant, by enough to swamp the engage/release hysteresis band. Engage and release timing was therefore partly a function of scrape routing. This replaces it with a single `limit=1` list of the workload namespace and computes `remainingItemCount + items.length`. One pod object transferred, no informer, no watch cache. Two request-shape constraints are load-bearing and called out in the code: passing a label or field selector makes the apiserver omit `remainingItemCount` entirely, and setting `resourceVersion` serves a cached count rather than a quorum read. Neither is passed. `remainingItemCount` is only set when the list is truncated, so `_continue` is the truncation signal — if it is absent the returned page is the whole collection and `items.length` is already exact. If the list *is* truncated and the count is missing or implausible, the fetcher throws rather than guessing. Failure semantics are unchanged: a throw lands in the monitor's existing catch, exactly as the previous parse did. The hysteresis, verdict shape, and gauge are untouched. RBAC is unchanged — the existing role already grants `pods: list`. The `/metrics` non-resource grant in the deployment role becomes unused, and the scrape-timeout env var is now a slight misnomer. Both left alone deliberately: the grant may be wanted again for other apiserver signals, and renaming the var would need a coordinated config change for no behavioural gain. Tests cover the not-truncated, truncated, missing-count, negative-count and timeout paths. --- .server-changes/pod-count-exact-list.md | 6 + .../k8sPodCountSignalSource.test.ts | 65 +++++---- .../backpressure/k8sPodCountSignalSource.ts | 20 +-- apps/supervisor/src/clients/kubernetes.ts | 126 +++++++++++------- apps/supervisor/src/index.ts | 8 +- 5 files changed, 124 insertions(+), 101 deletions(-) create mode 100644 .server-changes/pod-count-exact-list.md diff --git a/.server-changes/pod-count-exact-list.md b/.server-changes/pod-count-exact-list.md new file mode 100644 index 00000000000..d739f8d1209 --- /dev/null +++ b/.server-changes/pod-count-exact-list.md @@ -0,0 +1,6 @@ +--- +area: supervisor +type: improvement +--- + +Self-hosted Kubernetes deployments now measure running-task count more accurately when deciding whether to pause pulling new work, so the safeguard engages closer to its configured thresholds. diff --git a/apps/supervisor/src/backpressure/k8sPodCountSignalSource.test.ts b/apps/supervisor/src/backpressure/k8sPodCountSignalSource.test.ts index 8c7104cfb14..12cd92e2d88 100644 --- a/apps/supervisor/src/backpressure/k8sPodCountSignalSource.test.ts +++ b/apps/supervisor/src/backpressure/k8sPodCountSignalSource.test.ts @@ -1,52 +1,49 @@ import { describe, it, expect } from "vitest"; -import { parsePodCount, K8sPodCountSignalSource } from "./k8sPodCountSignalSource.js"; +import { K8sPodCountSignalSource } from "./k8sPodCountSignalSource.js"; +import { podCountFromList, withTimeout } from "../clients/kubernetes.js"; -describe("parsePodCount", () => { - it("reads the pods object count", () => { - const text = [ - "# HELP apiserver_storage_objects Number of stored objects", - "# TYPE apiserver_storage_objects gauge", - 'apiserver_storage_objects{resource="pods"} 8421', - 'apiserver_storage_objects{resource="configmaps"} 17', - ].join("\n"); - expect(parsePodCount(text)).toBe(8421); +describe("podCountFromList", () => { + it("returns items.length when the list is not truncated", () => { + expect(podCountFromList({ items: [{}], metadata: {} })).toBe(1); }); - it("is tolerant of extra labels in any order", () => { - const text = 'apiserver_storage_objects{group="",resource="pods",extra="x"} 12'; - expect(parsePodCount(text)).toBe(12); + it("returns zero for an empty namespace", () => { + expect(podCountFromList({ items: [], metadata: {} })).toBe(0); }); - it("parses scientific notation", () => { - const text = 'apiserver_storage_objects{resource="pods"} 1.2e+04'; - expect(parsePodCount(text)).toBe(12000); + it("adds remainingItemCount when the list is truncated", () => { + const list = { items: [{}], metadata: { _continue: "tok", remainingItemCount: 24492 } }; + expect(podCountFromList(list)).toBe(24493); }); - it("throws when the pods metric is absent", () => { - const text = 'apiserver_storage_objects{resource="configmaps"} 17'; - expect(() => parsePodCount(text)).toThrow(/not found/); + it("throws when truncated but remainingItemCount is absent", () => { + const list = { items: [{}], metadata: { _continue: "tok" } }; + expect(() => podCountFromList(list)).toThrow(/remainingItemCount/); }); - it("throws on a non-finite value (e.g. 1e999)", () => { - const text = 'apiserver_storage_objects{resource="pods"} 1e999'; - expect(() => parsePodCount(text)).toThrow(); + it("throws when truncated but remainingItemCount is negative", () => { + const list = { items: [{}], metadata: { _continue: "tok", remainingItemCount: -1 } }; + expect(() => podCountFromList(list)).toThrow(/remainingItemCount/); }); +}); - it("throws on a negative value", () => { - const text = 'apiserver_storage_objects{resource="pods"} -5'; - expect(() => parsePodCount(text)).toThrow(); +describe("withTimeout", () => { + it("rejects once the deadline passes", async () => { + await expect(withTimeout(new Promise(() => {}), 10, "pod count list")).rejects.toThrow( + /timed out/ + ); }); -}); -function metrics(count: number): string { - return `apiserver_storage_objects{resource="pods"} ${count}`; -} + it("passes a value through when it settles first", async () => { + await expect(withTimeout(Promise.resolve(7), 1000, "pod count list")).resolves.toBe(7); + }); +}); describe("K8sPodCountSignalSource", () => { it("engages at the engage threshold and reports the count", async () => { const counts: number[] = []; const source = new K8sPodCountSignalSource({ - fetchMetrics: async () => metrics(10000), + fetchPodCount: async () => 10000, engageThreshold: 10000, releaseThreshold: 5000, reportPodCount: (c) => counts.push(c), @@ -59,7 +56,7 @@ describe("K8sPodCountSignalSource", () => { it("does not engage below the engage threshold", async () => { const source = new K8sPodCountSignalSource({ - fetchMetrics: async () => metrics(9999), + fetchPodCount: async () => 9999, engageThreshold: 10000, releaseThreshold: 5000, }); @@ -69,7 +66,7 @@ describe("K8sPodCountSignalSource", () => { it("stays engaged in the hysteresis band, releases only below release threshold", async () => { let count = 10000; const source = new K8sPodCountSignalSource({ - fetchMetrics: async () => metrics(count), + fetchPodCount: async () => count, engageThreshold: 10000, releaseThreshold: 5000, }); @@ -82,9 +79,9 @@ describe("K8sPodCountSignalSource", () => { expect((await source.read()).engaged).toBe(false); // band again -> stays off }); - it("propagates scrape failures (monitor fails open on throw)", async () => { + it("propagates fetch failures (monitor fails open on throw)", async () => { const source = new K8sPodCountSignalSource({ - fetchMetrics: async () => { + fetchPodCount: async () => { throw new Error("connection refused"); }, engageThreshold: 10000, diff --git a/apps/supervisor/src/backpressure/k8sPodCountSignalSource.ts b/apps/supervisor/src/backpressure/k8sPodCountSignalSource.ts index e1fa0b78b11..949ebbfa025 100644 --- a/apps/supervisor/src/backpressure/k8sPodCountSignalSource.ts +++ b/apps/supervisor/src/backpressure/k8sPodCountSignalSource.ts @@ -1,22 +1,7 @@ import type { BackpressureSignalSource, BackpressureVerdict } from "./backpressureMonitor.js"; -// Reads the apiserver's stored-pod-object count from a Prometheus /metrics scrape. -const POD_COUNT_RE = /^apiserver_storage_objects\{[^}]*resource="pods"[^}]*\}\s+([0-9.eE+]+)/m; - -export function parsePodCount(metricsText: string): number { - const match = metricsText.match(POD_COUNT_RE); - if (!match) { - throw new Error('apiserver_storage_objects{resource="pods"} not found in metrics'); - } - const value = Number(match[1]); - if (!Number.isFinite(value)) { - throw new Error(`unparseable pod count: ${match[1]}`); - } - return value; -} - export type K8sPodCountSignalSourceOptions = { - fetchMetrics: () => Promise<string>; + fetchPodCount: () => Promise<number>; engageThreshold: number; releaseThreshold: number; reportPodCount?: (count: number) => void; @@ -29,8 +14,7 @@ export class K8sPodCountSignalSource implements BackpressureSignalSource { constructor(private readonly opts: K8sPodCountSignalSourceOptions) {} async read(): Promise<BackpressureVerdict> { - const text = await this.opts.fetchMetrics(); - const count = parsePodCount(text); + const count = await this.opts.fetchPodCount(); this.opts.reportPodCount?.(count); if (this.engaged) { diff --git a/apps/supervisor/src/clients/kubernetes.ts b/apps/supervisor/src/clients/kubernetes.ts index 9ab8e5bb3c1..1e511a68e6c 100644 --- a/apps/supervisor/src/clients/kubernetes.ts +++ b/apps/supervisor/src/clients/kubernetes.ts @@ -2,7 +2,6 @@ import * as k8s from "@kubernetes/client-node"; import type { Informer, KubernetesObject, ListPromise } from "@kubernetes/client-node"; import { assertExhaustive } from "@trigger.dev/core/utils"; import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; -import * as https from "node:https"; export const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local"; @@ -54,55 +53,90 @@ function getKubeConfig() { export { k8s }; /** - * Builds a function that scrapes the apiserver's Prometheus /metrics endpoint. - * One lightweight aggregate read - not a pod listing. Requires the service - * account to be granted GET on the /metrics non-resource URL. + * createPodCountFetcher sizes a namespace's pod collection with a single `limit=1` + * list: one pod transferred, no informer, no watch cache. + * + * This is an ESTIMATE, not an exact count. Kubernetes documents `remainingItemCount` + * as intended for estimating collection size and reserves the right not to set it or + * make it exact. Counting exactly would mean paginating the whole collection, which is + * what this deliberately avoids. Treat the value as a tight estimate from a quorum read + * at request time, and set thresholds with that in mind. + * + * Two request-shape constraints, both load-bearing. A label or field selector makes + * the apiserver omit `remainingItemCount` entirely, and setting `resourceVersion` + * serves a cached count instead of a quorum read - so neither is passed. */ -export function createApiserverMetricsFetcher(timeoutMs: number): () => Promise<string> { - const kubeConfig = getKubeConfig(); +export function createPodCountFetcher( + api: K8sApi, + namespace: string, + timeoutMs: number +): () => Promise<number> { + const serverTimeoutSeconds = Math.max(1, Math.floor(timeoutMs / 1000)); + let pending: Promise<unknown> | undefined; return async () => { - const cluster = kubeConfig.getCurrentCluster(); - if (!cluster) { - throw new Error("no current cluster in kubeconfig"); + if (pending) { + throw new Error("pod count list still in flight from a previous tick"); } - const url = new URL(`${cluster.server}/metrics`); - const opts: https.RequestOptions = { - method: "GET", - protocol: url.protocol, - hostname: url.hostname, - port: url.port, - path: url.pathname, - }; - // applyToHTTPSOptions sets the cluster CA, client cert/key, and auth headers - // (incl. exec plugins) on the request - so TLS verifies against the cluster - // CA, not the system store. The fetch-options path attaches the CA as an - // https.Agent, which global fetch (undici) ignores. - await kubeConfig.applyToHTTPSOptions(opts); - - return new Promise<string>((resolve, reject) => { - const req = https.request(opts, (res) => { - const status = res.statusCode ?? 0; - let body = ""; - res.setEncoding("utf8"); - res.on("data", (chunk) => { - body += chunk; - }); - res.on("end", () => { - if (status >= 200 && status < 300) { - resolve(body); - } else { - reject(new Error(`apiserver /metrics scrape failed: ${status}`)); - } - }); - }); - // Without this a hung connect/TLS/read never settles, and the monitor's - // refreshInFlight guard would freeze the source (silent fail-open). - req.setTimeout(timeoutMs, () => { - req.destroy(new Error(`apiserver /metrics scrape timed out after ${timeoutMs}ms`)); - }); - req.on("error", reject); - req.end(); + + const request = api.core.listNamespacedPod({ + namespace, + limit: 1, + timeoutSeconds: serverTimeoutSeconds, }); + + pending = request + .catch(() => {}) + .finally(() => { + pending = undefined; + }); + + return podCountFromList(await withTimeout(request, timeoutMs, "pod count list")); }; } + +/** + * podCountFromList turns a `limit=1` pod list into a population estimate. + * + * `remainingItemCount` is only set when the list is truncated, so `_continue` is the + * truncation signal: absent means the returned page is the whole collection and its + * length is exact. When truncated the total leans on `remainingItemCount`, which is + * documented as an estimate - so the result is an estimate too. Truncated without a + * usable count is unknowable, so it throws rather than returning a low number the + * caller would act on. + */ +export function podCountFromList(list: { + items: unknown[]; + metadata?: { _continue?: string; remainingItemCount?: number }; +}): number { + if (!list.metadata?._continue) { + return list.items.length; + } + + const remaining = list.metadata.remainingItemCount; + if (typeof remaining !== "number" || !Number.isFinite(remaining) || remaining < 0) { + throw new Error("pod list truncated but remainingItemCount absent or invalid"); + } + + return list.items.length + remaining; +} + +/** + * withTimeout rejects if `promise` outlives `timeoutMs`, so a hung request cannot + * freeze the caller. It cannot cancel: the k8s client threads no AbortSignal through to + * fetch, so an abandoned request keeps running. Callers must therefore also bound the + * request server-side (`timeoutSeconds`) and refuse to start a second one while the + * first is pending, or a blackholed connection accumulates one socket per attempt. + */ +export function withTimeout<T>(promise: Promise<T>, timeoutMs: number, what: string): Promise<T> { + let timer: NodeJS.Timeout; + const deadline = new Promise<never>((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error(`${what} timed out after ${timeoutMs}ms`)), + timeoutMs + ); + timer.unref(); + }); + + return Promise.race([promise, deadline]).finally(() => clearTimeout(timer)); +} diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index 39b9aa47e0a..542841bd6e5 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -21,7 +21,7 @@ import { CheckpointClient, isKubernetesEnvironment, } from "@trigger.dev/core/v3/serverOnly"; -import { createK8sApi, createApiserverMetricsFetcher } from "./clients/kubernetes.js"; +import { createK8sApi, createPodCountFetcher } from "./clients/kubernetes.js"; import { collectDefaultMetrics, Counter, Gauge, Histogram } from "prom-client"; import { register } from "./metrics.js"; import { PodCleaner } from "./services/podCleaner.js"; @@ -276,14 +276,16 @@ class ManagedSupervisor { // RELEASE < ENGAGE is enforced in env.ts (superRefine), so it's valid here. const podCountGauge = new Gauge({ name: "supervisor_cluster_pod_count", - help: "Total pod objects stored in the cluster, scraped for backpressure", + help: "Pod objects in the workload namespace, counted for backpressure", registers: [register], }); this.backpressureMonitors.push( new BackpressureMonitor({ enabled: true, source: new K8sPodCountSignalSource({ - fetchMetrics: createApiserverMetricsFetcher( + fetchPodCount: createPodCountFetcher( + createK8sApi(), + env.KUBERNETES_NAMESPACE, env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_SCRAPE_TIMEOUT_MS ), engageThreshold: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE, From 55e6225b0f60af7f4421fbfc48a25af294956c47 Mon Sep 17 00:00:00 2001 From: James Ritchie <james@trigger.dev> Date: Fri, 31 Jul 2026 21:57:53 +0100 Subject: [PATCH 50/67] fix(webapp): focus the search field when a filter sub-menu opens (#4443) ## Summary Opening a filter sub-menu that has its own search field left the cursor outside it, so you had to click into the field before you could type. The cursor now lands in the search field every time a sub-menu opens. `ComboBox` now focuses its input whenever the popover is open and the field is present, so the cursor lands there both when a menu opens normally and when a sub-menu mounts its field late. It is a no-op wherever focus already worked. Verified in the dashboard against the Tags menu: before, the field mounted with focus still on the popover container; after, it mounts focused and accepts typing straight away. --- apps/webapp/app/components/primitives/Select.tsx | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/apps/webapp/app/components/primitives/Select.tsx b/apps/webapp/app/components/primitives/Select.tsx index 3c4c8faad0c..0f592767ae6 100644 --- a/apps/webapp/app/components/primitives/Select.tsx +++ b/apps/webapp/app/components/primitives/Select.tsx @@ -694,6 +694,15 @@ export function ComboBox({ shortcut, ...props }: ComboBoxProps) { + const combobox = Ariakit.useComboboxContext(); + const open = combobox?.useState("open"); + const input = combobox?.useState("baseElement"); + + React.useEffect(() => { + if (!open || !input) return; + input.focus(); + }, [open, input]); + return ( <div className="flex h-9 w-full flex-none items-center border-b border-grid-dimmed bg-transparent pl-0 pr-3 text-xs text-text-dimmed outline-hidden"> <Ariakit.Combobox From fc691012521ee16c3204245bb229a42cbca3af11 Mon Sep 17 00:00:00 2001 From: James Ritchie <james@trigger.dev> Date: Fri, 31 Jul 2026 22:09:11 +0100 Subject: [PATCH 51/67] feat(webapp): AI agent logo experiments (#4399) ## Summary Adds an admin-only "AI agent" storybook page exploring an animated identity for the dashboard agent: a resting dot logo that animates while the agent is thinking, then settles once it is done. The lead experiment is a 5x5 dot matrix. Shapes are five-line string bitmaps, a bright head walks each shape's route on a fixed beat, and it only hands off between shapes on a dot the two share, so the rhythm never breaks. It comes with 26 faces, six gradient palettes, and light and dark treatments. Two earlier prototypes (a crisp logo that scatters into orbiting dots, and a dotted triangle on tilted 3D orbits) are kept in their own tabs for comparison. Everything is plain canvas code with no new dependencies. Also adds an `ask-ai` Button variant: secondary styling with a soft trigger-green border and padding tuned around the leading logo. The variant supplies the agent logo itself, so callers write `<Button variant="ask-ai/small">Ask AI</Button>`. Passing a `LeadingIcon` overrides it, which is how the thinking animation gets driven. No release note: the storybook is admin gated and the button variant is not used in product UI yet. --- .../components/primitives/AgentDotMatrix.tsx | 633 ++++++++ .../app/components/primitives/Buttons.tsx | 41 +- .../app/routes/storybook.ai-agent/route.tsx | 1267 +++++++++++++++++ .../app/routes/storybook.buttons/route.tsx | 20 + apps/webapp/app/routes/storybook/route.tsx | 4 + 5 files changed, 1963 insertions(+), 2 deletions(-) create mode 100644 apps/webapp/app/components/primitives/AgentDotMatrix.tsx create mode 100644 apps/webapp/app/routes/storybook.ai-agent/route.tsx diff --git a/apps/webapp/app/components/primitives/AgentDotMatrix.tsx b/apps/webapp/app/components/primitives/AgentDotMatrix.tsx new file mode 100644 index 00000000000..dd4484c29bb --- /dev/null +++ b/apps/webapp/app/components/primitives/AgentDotMatrix.tsx @@ -0,0 +1,633 @@ +import { type CSSProperties, useEffect, useRef } from "react"; + +// Our own 5x5 dot-matrix system, reverse-engineered from dotmatrix +// (github.com/zzzzshawn/matrix) but written from scratch on canvas. +// +// Their architecture, which we keep: +// - Shapes are masks over a fixed 5x5 grid of dots. +// - Animations are a bright "head" walking an ordered route through the +// shape's dots with a falling-off tail (their "Pulse Ladder" snake), over a +// dim base level. +// - Colors are 3-stop gradient presets with a separate glow color. +// +// What we change: +// - Shapes are authored as 5-line string bitmaps and the walk route is derived +// automatically (clockwise by angle from the center). +// - The gradient is evaluated ACROSS the grid at 135deg, so the matrix itself +// shows the sweep. +// - The full 5x5 grid is always visible: unlit cells are faint grey ghosts. +// - METRONOMIC transitions: the head never leaves the grid and never breaks +// its step rhythm. When a shape has run its cycles, the head keeps stepping +// until it lands on a dot SHARED with the next shape, then the next shape's +// route starts from that overlapped dot (route rotated to it) while the base +// dots crossfade. Same rule carries the head out of the rest logo and back +// into it. The default playlist is sequenced so every consecutive pair of +// shapes shares dots. + +export const MATRIX = 5; + +// --- shapes (5-line bitmaps: "o" = dot on) --------------------------------- + +export const DOT_SHAPES = { + // Rest faces — human, invader, cat, alien, robot, ghost. + face: [".....", ".o.o.", ".....", "o...o", ".ooo."], + invader: [".o.o.", "ooooo", "o.o.o", "ooooo", "o...o"], + cat: ["o...o", "ooooo", "o.o.o", "ooooo", ".ooo."], + alien: ["o...o", ".ooo.", "ooooo", "o.o.o", ".ooo."], + robot: ["ooooo", "o.o.o", "ooooo", "o...o", "ooooo"], + ghost: [".ooo.", "ooooo", "o.o.o", "ooooo", "o.o.o"], + // Extended face set. + owl: ["o...o", "ooooo", "o.o.o", ".ooo.", "..o.."], + dog: ["o...o", "ooooo", "o.o.o", "ooooo", "..o.."], + bunny: [".o.o.", ".o.o.", "ooooo", "o.o.o", ".ooo."], + bear: [".o.o.", "ooooo", "o.o.o", "ooooo", ".ooo."], + fox: ["o...o", "oo.oo", "ooooo", ".ooo.", "..o.."], + mouse: ["oo.oo", "ooooo", "o.o.o", ".ooo.", "..o.."], + koala: ["oo.oo", "ooooo", "o.o.o", "ooooo", ".ooo."], + frog: [".o.o.", ".ooo.", "ooooo", "o.o.o", "ooooo"], + penguin: [".ooo.", "ooooo", "o.o.o", "ooooo", "o...o"], + bat: ["o...o", "ooooo", "o.o.o", ".o.o.", "..o.."], + crab: ["o...o", "o.o.o", "ooooo", "ooooo", ".o.o."], + spider: ["o.o.o", ".ooo.", "ooooo", ".ooo.", "o.o.o"], + octopus: [".ooo.", "ooooo", "o.o.o", ".ooo.", "o.o.o"], + skull: [".ooo.", "ooooo", "o.o.o", ".ooo.", ".o.o."], + cyclops: [".ooo.", "ooooo", "oo.oo", ".ooo.", "..o.."], + sprite: [".ooo.", "ooooo", "o.o.o", ".ooo.", "..o.."], + clown: ["o.o.o", "ooooo", "o.o.o", "ooooo", ".ooo."], + mech: ["ooooo", "o.o.o", "ooooo", ".o.o.", "ooooo"], + angel: [".ooo.", ".....", "ooooo", "o.o.o", ".ooo."], + pumpkin: ["..o..", ".ooo.", "o.o.o", "ooooo", ".ooo."], + // Cycle shapes. + square: ["ooooo", "o...o", "o...o", "o...o", "ooooo"], + rectH: [".....", "ooooo", "o...o", "ooooo", "....."], + rectV: [".ooo.", ".o.o.", ".o.o.", ".o.o.", ".ooo."], + circle: [".ooo.", "o...o", "o...o", "o...o", ".ooo."], + diamond: ["..o..", ".o.o.", "o...o", ".o.o.", "..o.."], + heart: [".o.o.", "ooooo", "ooooo", ".ooo.", "..o.."], + checker: ["o.o.o", ".o.o.", "o.o.o", ".o.o.", "o.o.o"], +} satisfies Record<string, string[]>; + +export type DotShapeName = keyof typeof DOT_SHAPES; + +export const FACE_SHAPES: DotShapeName[] = ["face", "invader", "cat", "alien", "robot", "ghost"]; + +export const EXTRA_FACE_SHAPES: DotShapeName[] = [ + "owl", + "dog", + "bunny", + "bear", + "fox", + "mouse", + "koala", + "frog", + "penguin", + "bat", + "crab", + "spider", + "octopus", + "skull", + "cyclops", + "sprite", + "clown", + "mech", + "angel", + "pumpkin", +]; + +// Sequenced so every consecutive pair (including the wrap) shares dots — the +// head hands off between shapes without ever jumping. +export const DEFAULT_PLAYLIST: DotShapeName[] = [ + "square", + "rectH", + "circle", + "diamond", + "checker", + "rectV", + "heart", +]; + +// --- palettes (3-stop gradient + glow, like their color presets) ----------- + +export type DotMatrixPalette = { + /** 3 gradient stops, applied across the grid at 135deg. */ + stops: [string, string, string]; + /** Glow color for bright dots (defaults to the middle stop). */ + glow?: string; +}; + +export const DOT_MATRIX_PALETTES = { + mono: { stops: ["#e2e8f0", "#ffffff", "#94a3b8"], glow: "#ffffff" }, + trigger: { stops: ["#41ff54", "#a4ff53", "#e7ff52"], glow: "#86ff53" }, + aurora: { stops: ["#ff3cac", "#784ba0", "#2b86c5"], glow: "#9c64bf" }, + ocean: { stops: ["#00c6ff", "#0072ff", "#4facfe"], glow: "#2f8fff" }, + sunset: { stops: ["#ff5f6d", "#ffc371", "#ffe29a"], glow: "#ff8b73" }, + neon: { stops: ["#b4ff39", "#39ffb6", "#00d4ff"], glow: "#59ffc8" }, +} satisfies Record<string, DotMatrixPalette>; + +export type DotMatrixPaletteName = keyof typeof DOT_MATRIX_PALETTES; + +// --- geometry / precomputation ---------------------------------------------- + +type Rgb = [number, number, number]; + +function hexToRgb(hex: string): Rgb { + let h = hex.replace("#", ""); + if (h.length === 3) { + h = h[0] + h[0] + h[1] + h[1] + h[2] + h[2]; + } + const n = parseInt(h, 16); + return [(n >> 16) & 255, (n >> 8) & 255, n & 255]; +} + +const mix = (a: number, b: number, t: number) => a + (b - a) * t; +const mixRgb = (a: Rgb, b: Rgb, t: number): Rgb => [ + mix(a[0], b[0], t), + mix(a[1], b[1], t), + mix(a[2], b[2], t), +]; + +const smoothstep = (x: number) => { + const t = Math.min(1, Math.max(0, x)); + return t * t * (3 - 2 * t); +}; + +function parseShape(bitmap: string[]): boolean[] { + const mask = new Array<boolean>(MATRIX * MATRIX).fill(false); + for (let row = 0; row < MATRIX; row++) { + for (let col = 0; col < MATRIX; col++) { + if (bitmap[row]?.[col] === "o") { + mask[row * MATRIX + col] = true; + } + } + } + return mask; +} + +// Route = the shape's dots ordered clockwise by angle from the center starting +// at the top (ties: outer dots first). This is what the bright head walks. +function buildRoute(mask: boolean[]): number[] { + const center = (MATRIX - 1) / 2; + const cells: { index: number; angle: number; radius: number }[] = []; + for (let index = 0; index < mask.length; index++) { + if (!mask[index]) continue; + const row = Math.floor(index / MATRIX); + const col = index % MATRIX; + const dx = col - center; + const dy = row - center; + // 0 at 12 o'clock, increasing clockwise. + const angle = (Math.atan2(dx, -dy) + 2 * Math.PI) % (2 * Math.PI); + cells.push({ index, angle, radius: Math.hypot(dx, dy) }); + } + cells.sort((a, b) => a.angle - b.angle || b.radius - a.radius); + return cells.map((c) => c.index); +} + +// Rotate a cyclic route so it starts at `startCell` (falls back unrotated). +function rotateRoute(route: number[], startCell: number): number[] { + const at = route.indexOf(startCell); + if (at <= 0) return route; + return [...route.slice(at), ...route.slice(0, at)]; +} + +// Sample the 3-stop ramp at a continuous 0..1 position along the 135deg sweep. +function sampleRamp(stops: [Rgb, Rgb, Rgb], t: number): Rgb { + const clamped = Math.min(1, Math.max(0, t)); + return clamped <= 0.5 + ? mixRgb(stops[0], stops[1], clamped * 2) + : mixRgb(stops[1], stops[2], (clamped - 0.5) * 2); +} + +function paletteColors(stops: [Rgb, Rgb, Rgb]): Rgb[] { + const colors: Rgb[] = []; + for (let index = 0; index < MATRIX * MATRIX; index++) { + const row = Math.floor(index / MATRIX); + const col = index % MATRIX; + colors.push(sampleRamp(stops, (col / (MATRIX - 1) + row / (MATRIX - 1)) / 2)); + } + return colors; +} + +// Their Pulse Ladder falloff. Applied over the head's recent HISTORY (not +// route math), so the tail trails naturally through shape handoffs. +const SNAKE_TAIL = [1, 0.82, 0.68, 0.54, 0.42, 0.31, 0.22, 0.14]; + +// --- component --------------------------------------------------------------- + +export type AgentDotMatrixProps = { + /** Rendered size in CSS pixels. */ + size?: number; + /** false = static rest shape; true = animate through the playlist. */ + active?: boolean; + /** The resting shape (the "logo"). */ + restShape?: DotShapeName; + /** Shapes cycled through while thinking (consecutive shapes should overlap). */ + playlist?: DotShapeName[]; + /** Full route cycles each shape runs before handing off. */ + cyclesPerShape?: number; + /** 3-stop gradient palette (name or custom) used while thinking. */ + palette?: DotMatrixPaletteName | DotMatrixPalette; + /** Rest-state dot color; "palette" (default) themes the logo with the gradient. */ + restColor?: string | "palette"; + /** Opacity pair: dim base for shape dots, peak for the head/rest logo. */ + opacityBase?: number; + opacityPeak?: number; + /** + * Which background the matrix sits on. The ghost grid is white ink at low + * opacity for dark surfaces (reads as subtle grey) and black ink for light. + */ + mode?: "dark" | "light"; + /** Opacity of the ghost grid (0 disables it). */ + gridOpacity?: number; + /** + * When false (default), the ghost grid is transparent while resting (only + * the logo's dots show) and fades in while thinking. True: always visible. + */ + gridAtRest?: boolean; + /** Animation speed multiplier. */ + speed?: number; + /** + * Render as a decorative glyph (aria-hidden, no role) — use when the logo + * sits beside a text label that already names the control. + */ + decorative?: boolean; + className?: string; + style?: CSSProperties; + "aria-label"?: string; +}; + +const STEP_MS = 60; +const BLEND_MS = 240; + +export function AgentDotMatrix({ + size = 20, + active = false, + restShape = "alien", + playlist = DEFAULT_PLAYLIST, + cyclesPerShape = 2, + palette = "trigger", + restColor = "palette", + opacityBase = 0.3, + opacityPeak = 0.95, + mode = "dark", + gridOpacity = 0.18, + gridAtRest = false, + speed = 1, + decorative = false, + className, + style, + "aria-label": ariaLabel, +}: AgentDotMatrixProps) { + const canvasRef = useRef<HTMLCanvasElement>(null); + const activeRef = useRef(active); + const wakeRef = useRef<() => void>(() => {}); + + const paletteObj: DotMatrixPalette = + typeof palette === "string" ? DOT_MATRIX_PALETTES[palette] : palette; + const paletteKey = paletteObj.stops.join(",") + (paletteObj.glow ?? ""); + const playlistKey = playlist.join(","); + + useEffect(() => { + activeRef.current = active; + wakeRef.current(); + }, [active]); + + useEffect(() => { + const canvas = canvasRef.current; + const ctx = canvas?.getContext("2d"); + if (!canvas || !ctx) return; + + const dpr = Math.min(2, typeof devicePixelRatio !== "undefined" ? devicePixelRatio : 1); + canvas.width = Math.round(size * dpr); + canvas.height = Math.round(size * dpr); + + const shapeNames: DotShapeName[] = [restShape, ...playlist]; + const masks = new Map<DotShapeName, boolean[]>(); + const routes = new Map<DotShapeName, number[]>(); + for (const name of shapeNames) { + const mask = parseShape(DOT_SHAPES[name]); + masks.set(name, mask); + routes.set(name, buildRoute(mask)); + } + const stops = paletteObj.stops.map(hexToRgb) as [Rgb, Rgb, Rgb]; + const gridColors = paletteColors(stops); + const glowRgb = hexToRgb(paletteObj.glow ?? paletteObj.stops[1]); + const restRgb: Rgb[] = + restColor === "palette" ? gridColors : new Array(MATRIX * MATRIX).fill(hexToRgb(restColor)); + const restMask = masks.get(restShape)!; + + const pitch = size / MATRIX; + const dotR = Math.max(0.75, pitch * 0.3); + // Clamp: a zero or negative speed would make stepMs non-positive and spin + // the step loop forever. + const safeSpeed = Math.max(0.01, speed); + const stepMs = STEP_MS / safeSpeed; + const blendMs = BLEND_MS / safeSpeed; + // White ink at low opacity reads as subtle grey on any dark background; + // light mode flips to black ink. + const gridRgb: Rgb = mode === "light" ? [0, 0, 0] : [255, 255, 255]; + + // --- engine state (refs only; no React re-renders) --- + // The head is metronomic: while running it moves exactly one route dot per + // step, including straight through shape handoffs (shared dot = no jump). + let running_shape = false; // false = at rest (no head) + let route: number[] = []; // current (rotated) route + let playlistIdx = -1; + let headPos = 0; // index into route + let stepsInShape = 0; + let stepClock = 0; + let seeking: "next" | "rest" | null = null; + let seekSteps = 0; + const history: number[] = []; // recent head cells, newest first + let tailFade = 1; // ramps to 0 while settling back to rest + let gridFade = gridAtRest ? 1 : 0; // ghost-grid visibility (fades with activity) + let glowFade = 0; // dot glow only while animating — static logos have no glow + // Base crossfade between the previous and current shape's dim base maps. + let blendT = 1; + const baseFrom = new Float64Array(MATRIX * MATRIX); + const blendFrom = new Float64Array(MATRIX * MATRIX); + const baseTo = new Float64Array(MATRIX * MATRIX); + const blendTo = new Float64Array(MATRIX * MATRIX); + + const setTargetMaps = (m: boolean[], level: number, colorBlend: number) => { + for (let i = 0; i < baseTo.length; i++) { + baseTo[i] = m[i] ? level : 0; + blendTo[i] = m[i] ? colorBlend : 0; + } + }; + + const snapshotToFrom = () => { + const t = smoothstep(blendT); + for (let i = 0; i < baseFrom.length; i++) { + baseFrom[i] = mix(baseFrom[i], baseTo[i], t); + blendFrom[i] = mix(blendFrom[i], blendTo[i], t); + } + blendT = 0; + }; + + // Initial pose: rest logo, fully blended. + setTargetMaps(restMask, opacityPeak, 0); + baseFrom.set(baseTo); + blendFrom.set(blendTo); + + // Shared-dot handoff: rotate the target's route to start at the head's + // current cell (guaranteed shared by the playlist sequencing; nearest-dot + // fallback keeps custom configs safe). + const handoffTo = (target: DotShapeName, targetIdx: number, atCell: number) => { + const targetMask = masks.get(target)!; + const canonical = routes.get(target)!; + const startCell = targetMask[atCell] + ? atCell + : canonical.reduce( + (best, cell) => + Math.hypot( + (cell % MATRIX) - (atCell % MATRIX), + Math.floor(cell / MATRIX) - Math.floor(atCell / MATRIX) + ) < + Math.hypot( + (best % MATRIX) - (atCell % MATRIX), + Math.floor(best / MATRIX) - Math.floor(atCell / MATRIX) + ) + ? cell + : best, + canonical[0] + ); + route = rotateRoute(canonical, startCell); + playlistIdx = targetIdx; + headPos = 0; + stepsInShape = 0; + seeking = null; + seekSteps = 0; + snapshotToFrom(); + setTargetMaps(targetMask, opacityBase, 1); + if (startCell !== history[0]) { + history.unshift(startCell); + if (history.length > SNAKE_TAIL.length) history.length = SNAKE_TAIL.length; + } + }; + + const settleToRest = () => { + running_shape = false; + playlistIdx = -1; + seeking = null; + snapshotToFrom(); + setTargetMaps(restMask, opacityPeak, 0); + }; + + const beginThinking = () => { + const first = playlist[0]; + // Spawn the head on a dot shared between the rest logo and the first + // shape — earliest such dot in the shape's canonical route. + const canonical = routes.get(first)!; + const spawn = canonical.find((cell) => restMask[cell]) ?? canonical[0]; + history.length = 0; + history.unshift(spawn); + tailFade = 1; + running_shape = true; + handoffTo(first, 0, spawn); + }; + + // Draw in device-pixel space with centers/radii snapped to whole device + // pixels — dot positions never animate (only opacity), and unsnapped + // fractional centers (e.g. 14px -> 2.8px pitch) blur every dot differently. + // Light mode gets a size boost (snapped to half pixels): dark-on-light dots + // read optically smaller than the same-size light-on-dark dots. + const devCenter = (cell: number) => Math.round((cell + 0.5) * pitch * dpr); + const devDotR = + mode === "light" + ? Math.max(1, Math.round(dotR * dpr * 1.15 * 2) / 2) + : Math.max(1, Math.round(dotR * dpr)); + + const draw = () => { + ctx.setTransform(1, 0, 0, 1, 0, 0); + ctx.clearRect(0, 0, canvas.width, canvas.height); + const t = smoothstep(blendT); + for (let index = 0; index < MATRIX * MATRIX; index++) { + let alpha = mix(baseFrom[index], baseTo[index], t); + let cBlend = mix(blendFrom[index], blendTo[index], t); + // Tail overlay from the head's recent history (trails through handoffs). + if (tailFade > 0.01) { + const d = history.indexOf(index); + if (d >= 0 && d < SNAKE_TAIL.length) { + const level = mix(opacityBase, opacityPeak, SNAKE_TAIL[d]) * tailFade; + if (level > alpha) { + alpha = level; + cBlend = Math.max(cBlend, tailFade); + } + } + } + const row = Math.floor(index / MATRIX); + const col = index % MATRIX; + const x = devCenter(col); + const y = devCenter(row); + ctx.beginPath(); + if (alpha <= gridOpacity) { + const ghostAlpha = gridOpacity * gridFade; + if (ghostAlpha <= 0.004) continue; + ctx.shadowBlur = 0; + ctx.fillStyle = `rgba(${gridRgb[0]},${gridRgb[1]},${gridRgb[2]},${ghostAlpha.toFixed(3)})`; + ctx.arc(x, y, devDotR, 0, Math.PI * 2); + ctx.fill(); + continue; + } + const rgb = mixRgb(restRgb[index], gridColors[index], cBlend); + ctx.fillStyle = `rgba(${rgb[0] | 0},${rgb[1] | 0},${rgb[2] | 0},${alpha.toFixed(3)})`; + const glowAlpha = alpha >= 0.6 ? ((alpha - 0.6) / 0.4) * glowFade : 0; + if (glowAlpha > 0.01) { + ctx.shadowColor = `rgba(${glowRgb[0]},${glowRgb[1]},${glowRgb[2]},${glowAlpha.toFixed(3)})`; + ctx.shadowBlur = devDotR * 2.2; + } else { + ctx.shadowBlur = 0; + } + ctx.arc(x, y, devDotR, 0, Math.PI * 2); + ctx.fill(); + } + ctx.shadowBlur = 0; + }; + + const reduced = + typeof matchMedia !== "undefined" && matchMedia("(prefers-reduced-motion: reduce)").matches; + if (reduced) { + draw(); + return; + } + + let raf = 0; + let running = false; + let last = 0; + + // One metronome step: move the head one dot, then apply the transition + // rules — (1) enough cycles? (2) on a shared dot? (3) hand off from it. + const stepOnce = () => { + headPos = (headPos + 1) % route.length; + stepsInShape++; + history.unshift(route[headPos]); + if (history.length > SNAKE_TAIL.length) history.length = SNAKE_TAIL.length; + + // Every change of seek mode restarts the step budget, so each search gets + // a full lap to find a shared dot before its fallback fires. + if (!activeRef.current) { + if (seeking !== "rest") { + seeking = "rest"; + seekSteps = 0; + } + } else { + if (seeking === "rest") { + // Re-activated mid-settle: cancel the pending return to rest so a + // quick off/on toggle carries on instead of restarting. + seeking = null; + seekSteps = 0; + } + if (seeking !== "next" && stepsInShape >= cyclesPerShape * route.length) { + seeking = "next"; + seekSteps = 0; + } + } + if (!seeking) return; + + seekSteps++; + const headCell = route[headPos]; + if (seeking === "rest") { + if (restMask[headCell] || seekSteps > route.length + 1) { + settleToRest(); + } + } else { + const nextIdx = (playlistIdx + 1) % playlist.length; + const nextMask = masks.get(playlist[nextIdx])!; + if (nextMask[headCell] || seekSteps > route.length + 1) { + handoffTo(playlist[nextIdx], nextIdx, headCell); + } + } + }; + + const frame = (now: number) => { + const dt = Math.min(80, now - last); + last = now; + let keepGoing = true; + + // The ghost grid follows activity when gridAtRest is off; the glow always does. + const gridTarget = gridAtRest || running_shape ? 1 : 0; + if (gridFade < gridTarget) { + gridFade = Math.min(gridTarget, gridFade + dt / blendMs); + } else if (gridFade > gridTarget) { + gridFade = Math.max(gridTarget, gridFade - dt / blendMs); + } + const glowTarget = running_shape ? 1 : 0; + if (glowFade < glowTarget) { + glowFade = Math.min(glowTarget, glowFade + dt / blendMs); + } else if (glowFade > glowTarget) { + glowFade = Math.max(glowTarget, glowFade - dt / blendMs); + } + + if (!running_shape) { + if (activeRef.current && playlist.length > 0) { + beginThinking(); + } else { + // Settling home: finish the crossfade, fade the tail + grid + glow, then stop. + blendT = Math.min(1, blendT + dt / blendMs); + tailFade = Math.max(0, tailFade - dt / blendMs); + if (blendT >= 1 && tailFade <= 0 && gridFade === gridTarget && glowFade === 0) { + keepGoing = false; + } + } + } + + if (running_shape) { + blendT = Math.min(1, blendT + dt / blendMs); + stepClock += dt; + while (stepClock >= stepMs && running_shape) { + stepClock -= stepMs; + stepOnce(); + } + } + + draw(); + + if (keepGoing) { + raf = requestAnimationFrame(frame); + } else { + running = false; + } + }; + + const wake = () => { + if (running) return; + running = true; + last = typeof performance !== "undefined" ? performance.now() : 0; + raf = requestAnimationFrame(frame); + }; + wakeRef.current = wake; + + draw(); + wake(); + + return () => { + running = false; + wakeRef.current = () => {}; + cancelAnimationFrame(raf); + }; + }, [ + size, + speed, + restShape, + playlistKey, + cyclesPerShape, + paletteKey, + restColor, + opacityBase, + opacityPeak, + mode, + gridOpacity, + gridAtRest, + ]); + + return ( + <canvas + ref={canvasRef} + {...(decorative + ? { "aria-hidden": true } + : { role: "img", "aria-label": ariaLabel ?? (active ? "Agent thinking" : "Agent") })} + className={className} + style={{ width: size, height: size, display: "block", ...style }} + /> + ); +} diff --git a/apps/webapp/app/components/primitives/Buttons.tsx b/apps/webapp/app/components/primitives/Buttons.tsx index 4f4b694bccc..792f73eb595 100644 --- a/apps/webapp/app/components/primitives/Buttons.tsx +++ b/apps/webapp/app/components/primitives/Buttons.tsx @@ -9,6 +9,7 @@ import React, { } from "react"; import { type ShortcutDefinition, useShortcutKeys } from "~/hooks/useShortcutKeys"; import { cn } from "~/utils/cn"; +import { AgentDotMatrix } from "./AgentDotMatrix"; import { ShortcutKey } from "./ShortcutKey"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "./Tooltip"; import { Icon, type RenderIcon } from "./Icon"; @@ -113,6 +114,16 @@ const theme = { "border-text-dimmed/40 text-text-dimmed group-hover/button:text-text-bright group-hover/button:border-text-dimmed", icon: "text-blue-500", }, + // Reserved for the AI agent's "Ask AI" affordance: secondary styling with a + // softened trigger-green border. + "ask-ai": { + textColor: "text-text-bright transition group-disabled/button:text-text-dimmed/80", + button: + "cursor-pointer bg-secondary border border-[#41FF54]/25 group-hover/button:bg-surface-control group-hover/button:border-[#41FF54]/40 group-disabled/button:bg-secondary group-disabled/button:opacity-60 group-disabled/button:cursor-default group-disabled/button:pointer-events-none", + shortcut: + "border-text-dimmed/40 text-text-dimmed group-hover/button:text-text-bright group-hover/button:border-text-dimmed", + icon: "text-text-bright", + }, }; type Theme = keyof typeof theme; @@ -125,6 +136,24 @@ function createVariant(sizeName: Size, themeName: Theme) { iconSpacing: sizes[sizeName].iconSpacing, shortcutVariant: sizes[sizeName].shortcutVariant, shortcut: cn(sizes[sizeName].shortcut, theme[themeName].shortcut), + // Rendered as the leading icon when the caller doesn't pass one. + defaultLeadingIcon: undefined as RenderIcon | undefined, + }; +} + +// The ask-ai button always leads with the square agent logo, so it supplies its +// own leading icon and its padding is tuned around it: small = 16px logo, 4px +// left / 6px right; medium 32/16 -> 8px; large 40/20 -> 10px. Pass an explicit +// `LeadingIcon` (e.g. an <AgentDotMatrix active />) to animate it. +function createAskAiVariant(sizeName: Size, opticalPadding: string, logoSize: number) { + const base = createVariant(sizeName, "ask-ai"); + return { + ...base, + button: cn(base.button, opticalPadding), + iconSpacing: "gap-x-1.5", + defaultLeadingIcon: ( + <AgentDotMatrix size={logoSize} palette="mono" restColor="#ffffff" decorative /> + ), }; } @@ -158,6 +187,9 @@ const variant = { "docs/medium": createVariant("medium", "docs"), "docs/large": createVariant("large", "docs"), "docs/extra-large": createVariant("extra-large", "docs"), + "ask-ai/small": createAskAiVariant("small", "px-1 pr-1.5", 16), + "ask-ai/medium": createAskAiVariant("medium", "px-2", 16), + "ask-ai/large": createAskAiVariant("large", "px-2.5", 20), "menu-item": { textColor: "text-text-bright px-1", button: @@ -166,6 +198,7 @@ const variant = { iconSpacing: "gap-x-0.5", shortcutVariant: undefined, shortcut: undefined, + defaultLeadingIcon: undefined, }, "small-menu-item": { textColor: "text-text-bright", @@ -175,6 +208,7 @@ const variant = { iconSpacing: "gap-x-1.5", shortcutVariant: undefined, shortcut: undefined, + defaultLeadingIcon: undefined, }, "small-menu-sub-item": { textColor: "text-text-dimmed", @@ -184,6 +218,7 @@ const variant = { iconSpacing: undefined, shortcutVariant: undefined, shortcut: undefined, + defaultLeadingIcon: undefined, }, }; @@ -240,6 +275,8 @@ export function ButtonContent(props: ButtonContentPropsType) { }, [isLoading]); const variation = allVariants.variant[props.variant]; + // Some variants (ask-ai) always lead with their own glyph unless overridden. + const leadingIcon = LeadingIcon ?? variation.defaultLeadingIcon; const btnClassName = cn(allVariants.$all, variation.button); const iconClassName = variation.icon; @@ -269,9 +306,9 @@ export function ButtonContent(props: ButtonContentPropsType) { showSpinner && "invisible" )} > - {LeadingIcon && ( + {leadingIcon && ( <Icon - icon={LeadingIcon} + icon={leadingIcon} className={cn( iconClassName, variation.icon, diff --git a/apps/webapp/app/routes/storybook.ai-agent/route.tsx b/apps/webapp/app/routes/storybook.ai-agent/route.tsx new file mode 100644 index 00000000000..76605c33462 --- /dev/null +++ b/apps/webapp/app/routes/storybook.ai-agent/route.tsx @@ -0,0 +1,1267 @@ +import { type CSSProperties, useEffect, useMemo, useRef, useState } from "react"; +import { LogoIcon } from "~/components/LogoIcon"; +import { Button, type ButtonVariant } from "~/components/primitives/Buttons"; +import { + ClientTabs, + ClientTabsContent, + ClientTabsList, + ClientTabsTrigger, +} from "~/components/primitives/ClientTabs"; +import { Header1, Header2 } from "~/components/primitives/Headers"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { + AgentDotMatrix, + DOT_MATRIX_PALETTES, + DOT_SHAPES, + EXTRA_FACE_SHAPES, + FACE_SHAPES, + type DotMatrixPaletteName, + type DotShapeName, +} from "~/components/primitives/AgentDotMatrix"; + +// Experiments for the trigger.dev AI dashboard-agent identity: a resting logo +// that animates while the agent thinks. Each tab is a separate experiment. + +export default function Story() { + return ( + <div className="flex flex-col gap-4 p-6"> + <div className="flex max-w-3xl flex-col gap-1"> + <Header1>AI agent</Header1> + <Paragraph variant="small"> + A resting logo that animates while the agent thinks. Each tab is a separate experiment. + </Paragraph> + </div> + <ClientTabs defaultValue="dot-matrix"> + <ClientTabsList variant="underline"> + <ClientTabsTrigger variant="underline" value="dot-matrix"> + Dot matrix + </ClientTabsTrigger> + <ClientTabsTrigger variant="underline" value="logo-morph"> + Logo morph + </ClientTabsTrigger> + <ClientTabsTrigger variant="underline" value="orbit-dots"> + Orbit dots + </ClientTabsTrigger> + </ClientTabsList> + <ClientTabsContent value="dot-matrix"> + <DotMatrixTab /> + </ClientTabsContent> + <ClientTabsContent value="logo-morph"> + <LogoMorphTab /> + </ClientTabsContent> + <ClientTabsContent value="orbit-dots"> + <OrbitDotsTab /> + </ClientTabsContent> + </ClientTabs> + </div> + ); +} + +// --- Dot matrix (5x5) --------------------------------------------------------- + +// Dark-ink mono ramp for light surfaces (the built-in mono palette is white-based). +const LIGHT_MONO = { + stops: ["#0d0e12", "#1a1b1f", "#3b3e45"] as [string, string, string], + glow: "#1a1b1f", +}; + +function DotMatrixTab() { + return ( + <div className="flex flex-col gap-6 py-6"> + <Paragraph variant="small" className="mt-2 -mb-3 max-w-3xl"> + 5×5 grid. A bright head walks each shape's route on a fixed beat, 2 cycles per shape, + handing off at shared dots. Click to toggle. + </Paragraph> + <div className="flex flex-wrap items-end gap-8 rounded-md border border-grid-bright bg-background-bright px-6 py-5"> + {[16, 20, 24, 32, 48].map((s) => ( + <div key={s} className="flex flex-col items-center gap-2"> + <div className="flex h-12 items-center"> + <AgentDotMatrix size={s} /> + </div> + <div className="text-[10px] uppercase tracking-wide text-text-dimmed">{s}px rest</div> + </div> + ))} + </div> + <Paragraph variant="small" className="mt-2 -mb-3 max-w-3xl"> + The <code className="text-text-bright">ask-ai</code> Button variant. Mono logo, click to + think for 5s. + </Paragraph> + <div className="flex flex-wrap items-center gap-6 rounded-md border border-grid-bright bg-background-bright px-6 py-5"> + {( + [ + ["ask-ai/small", 16, "small"], + ["ask-ai/medium", 16, "medium"], + ["ask-ai/large", 20, "large"], + ] as [ButtonVariant, number, string][] + ).map(([variant, matrixSize, label]) => ( + <div key={variant} className="flex flex-col items-center gap-2"> + <AskAiButton variant={variant} matrixSize={matrixSize} /> + <div className="text-[10px] uppercase tracking-wide text-text-dimmed">{label}</div> + </div> + ))} + </div> + <Paragraph variant="small" className="mt-2 -mb-3 max-w-3xl"> + Small-button icon size comparison (crispness): + </Paragraph> + <div className="flex flex-wrap items-center gap-6 rounded-md border border-grid-bright bg-background-bright px-6 py-5"> + {[14, 15, 16].map((s) => ( + <div key={s} className="flex flex-col items-center gap-2"> + <AskAiButton variant="ask-ai/small" matrixSize={s} /> + <div className="text-[10px] uppercase tracking-wide text-text-dimmed">{s}px icon</div> + </div> + ))} + </div> + <div className="flex flex-wrap items-end gap-10 rounded-md border border-grid-bright bg-background-bright px-6 py-6"> + {[20, 32, 48, 72].map((s) => ( + <div key={s} className="flex flex-col items-center gap-2"> + <ToggleableMatrix size={s} /> + <div className="text-[10px] uppercase tracking-wide text-text-dimmed"> + {s}px · click + </div> + </div> + ))} + </div> + <Paragraph variant="small" className="mt-2 -mb-3 max-w-3xl"> + Mono on dark (charcoal-800) and light (charcoal-100): + </Paragraph> + <div className="flex flex-wrap gap-4"> + <div className="flex items-center gap-8 rounded-md border border-grid-bright bg-charcoal-800 px-6 py-5"> + <AgentDotMatrix size={40} mode="dark" palette="mono" restColor="#d7d9dd" /> + <ToggleableMatrix + size={40} + matrix={{ mode: "dark", palette: "mono", restColor: "#d7d9dd" }} + /> + </div> + <div className="flex items-center gap-8 rounded-md border border-grid-bright bg-charcoal-100 px-6 py-5"> + <AgentDotMatrix size={40} mode="light" palette={LIGHT_MONO} restColor="#1a1b1f" /> + <ToggleableMatrix + size={40} + matrix={{ mode: "light", palette: LIGHT_MONO, restColor: "#1a1b1f" }} + /> + </div> + </div> + <Paragraph variant="small" className="mt-2 -mb-3 max-w-3xl"> + Face suite (small buttons, click to think): + </Paragraph> + <div className="flex max-w-4xl flex-wrap gap-3 rounded-md border border-grid-bright bg-background-bright px-6 py-5"> + {EXTRA_FACE_SHAPES.map((name) => ( + <FaceButton key={name} name={name} /> + ))} + </div> + <Paragraph variant="small" className="mt-2 -mb-3 max-w-3xl"> + Face options (grid always visible here): + </Paragraph> + <div className="flex flex-wrap items-end gap-8 rounded-md border border-grid-bright bg-background-bright px-6 py-5"> + {FACE_SHAPES.map((name) => ( + <div key={name} className="flex flex-col items-center gap-2"> + <ToggleableMatrix size={40} matrix={{ restShape: name, gridAtRest: true }} /> + <div className="text-[10px] uppercase tracking-wide text-text-dimmed">{name}</div> + </div> + ))} + </div> + <Paragraph variant="small" className="mt-2 -mb-3 max-w-3xl"> + Palettes (always running): + </Paragraph> + <div className="flex flex-wrap items-end gap-8 rounded-md border border-grid-bright bg-background-bright px-6 py-5"> + {(Object.keys(DOT_MATRIX_PALETTES) as DotMatrixPaletteName[]).map((name) => ( + <div key={name} className="flex flex-col items-center gap-2"> + <AgentDotMatrix size={48} active palette={name} restColor="palette" /> + <div className="text-[10px] uppercase tracking-wide text-text-dimmed">{name}</div> + </div> + ))} + </div> + <Paragraph variant="small" className="mt-2 -mb-3 max-w-3xl"> + Shape library (5-line string bitmaps): + </Paragraph> + <div className="flex flex-wrap items-end gap-8 rounded-md border border-grid-bright bg-background-bright px-6 py-5"> + {(Object.keys(DOT_SHAPES) as DotShapeName[]).map((name) => ( + <div key={name} className="flex flex-col items-center gap-2"> + <AgentDotMatrix size={32} restShape={name} /> + <div className="text-[10px] uppercase tracking-wide text-text-dimmed">{name}</div> + </div> + ))} + </div> + </div> + ); +} + +function AskAiButton({ variant, matrixSize }: { variant: ButtonVariant; matrixSize: number }) { + const [active, setActive] = useState(false); + const timeout = useRef<ReturnType<typeof setTimeout>>(); + + useEffect(() => () => clearTimeout(timeout.current), []); + + const trigger = () => { + setActive(true); + clearTimeout(timeout.current); + timeout.current = setTimeout(() => setActive(false), 5000); + }; + + // Overrides the variant's built-in static logo so it can animate. + return ( + <Button + variant={variant} + onClick={trigger} + LeadingIcon={ + <AgentDotMatrix + size={matrixSize} + active={active} + palette="mono" + restColor="#ffffff" + decorative + /> + } + > + Ask AI + </Button> + ); +} + +function FaceButton({ name }: { name: DotShapeName }) { + const [active, setActive] = useState(false); + const timeout = useRef<ReturnType<typeof setTimeout>>(); + + useEffect(() => () => clearTimeout(timeout.current), []); + + const trigger = () => { + setActive(true); + clearTimeout(timeout.current); + timeout.current = setTimeout(() => setActive(false), 5000); + }; + + return ( + <Button + variant="ask-ai/small" + onClick={trigger} + LeadingIcon={ + <AgentDotMatrix + size={16} + active={active} + restShape={name} + palette="mono" + restColor="#ffffff" + decorative + /> + } + > + {name} + </Button> + ); +} + +function ToggleableMatrix({ + size, + matrix, +}: { + size: number; + matrix?: Partial<React.ComponentProps<typeof AgentDotMatrix>>; +}) { + const [active, setActive] = useState(false); + + return ( + <button + type="button" + onClick={() => setActive((a) => !a)} + aria-label={active ? "Stop the thinking animation" : "Start the thinking animation"} + aria-pressed={active} + title="Click to toggle thinking" + className="focus-custom cursor-pointer rounded-md p-2 transition hover:bg-background-hover" + > + <AgentDotMatrix size={size} active={active} {...matrix} /> + </button> + ); +} + +// --- Logo morph (crisp logo -> orbits) ----------------------------------------- + +function LogoMorphTab() { + return ( + <div className="flex flex-col gap-6 py-6"> + <Paragraph variant="small" className="mt-2 -mb-3 max-w-3xl"> + Crisp logo at rest; dots born on its outline fly out to orbits. Click or scrub. + </Paragraph> + <div className="flex flex-wrap items-end gap-8 rounded-md border border-grid-bright bg-background-bright px-6 py-5"> + {[16, 20, 24, 32, 48].map((s) => ( + <div key={s} className="flex flex-col items-center gap-2"> + <div className="flex h-12 items-center"> + <AgentLogoMorph size={s} /> + </div> + <div className="text-[10px] uppercase tracking-wide text-text-dimmed">{s}px rest</div> + </div> + ))} + </div> + <div className="flex flex-wrap items-end gap-10 rounded-md border border-grid-bright bg-background-bright px-6 py-6"> + {[20, 32, 48, 72].map((s) => ( + <div key={s} className="flex flex-col items-center gap-2"> + <TriggerableLogoMorph size={s} /> + <div className="text-[10px] uppercase tracking-wide text-text-dimmed"> + {s}px · click + </div> + </div> + ))} + </div> + <LogoMorphScrubber size={140} /> + </div> + ); +} + +function TriggerableLogoMorph({ size }: { size: number }) { + const [active, setActive] = useState(false); + const timeout = useRef<ReturnType<typeof setTimeout>>(); + + useEffect(() => () => clearTimeout(timeout.current), []); + + const trigger = () => { + setActive(true); + clearTimeout(timeout.current); + timeout.current = setTimeout(() => setActive(false), 4000); + }; + + return ( + <button + type="button" + onClick={trigger} + aria-label="Trigger the agent thinking animation" + title="Click to think" + className="focus-custom cursor-pointer rounded-md p-2 transition hover:bg-background-hover" + > + <AgentLogoMorph size={size} active={active} /> + </button> + ); +} + +function LogoMorphScrubber({ size = 140 }: { size?: number }) { + const [value, setValue] = useState(0); + + return ( + <div className="flex flex-col items-start gap-4 rounded-md border border-grid-bright bg-background-bright px-6 py-6"> + <AgentLogoMorph size={size} activation={value} /> + <div className="flex w-full max-w-md items-center gap-3"> + <span className="w-10 text-xs tabular-nums text-text-dimmed">{value.toFixed(2)}</span> + <input + type="range" + min={0} + max={1} + step={0.01} + value={value} + onChange={(e) => setValue(parseFloat(e.target.value))} + aria-label="Morph amount" + className="h-1 w-full cursor-pointer appearance-none rounded-full bg-grid-bright accent-blue-500" + /> + </div> + <div className="flex w-full max-w-md justify-between text-[10px] uppercase tracking-wide text-text-dimmed"> + <span>Logo</span> + <span>Working</span> + </div> + </div> + ); +} + +// --- Orbit dots (triangle -> tilted 3D orbits) ---------------------------------- + +function OrbitDotsTab() { + return ( + <div className="flex flex-col gap-6 py-6"> + <Paragraph variant="small" className="mt-2 -mb-3 max-w-3xl"> + Dotted triangle ↔ tilted 3D orbits. White at rest, palette while working. + </Paragraph> + <section className="flex flex-col gap-3"> + <Header2>Static logo</Header2> + <div className="flex flex-wrap items-end gap-8 rounded-md border border-grid-bright bg-background-bright px-6 py-5"> + {[16, 20, 24, 32, 48].map((s) => ( + <div key={s} className="flex flex-col items-center gap-2"> + <div className="flex h-12 items-center"> + <AgentOrb size={s} adaptive /> + </div> + <div className="text-[10px] uppercase tracking-wide text-text-dimmed">{s}px</div> + </div> + ))} + <div className="flex items-center gap-3 self-center"> + <div className="flex items-center gap-2 rounded-md bg-tertiary px-3 py-2 text-sm text-text-bright"> + <AgentOrb size={20} adaptive /> + Ask the agent + </div> + <div className="flex items-center gap-2 rounded-md bg-tertiary px-3 py-2 text-sm text-text-bright"> + <TriggerTriangle className="h-4 w-auto text-green-500" /> + Ask the agent + </div> + </div> + </div> + </section> + <section className="flex flex-col gap-3"> + <Header2>Click to think</Header2> + <div className="flex flex-wrap items-end gap-10 rounded-md border border-grid-bright bg-background-bright px-6 py-6"> + {[20, 32, 48, 72].map((s) => ( + <div key={s} className="flex flex-col items-center gap-2"> + <TriggerableOrb size={s} orb={{ adaptive: true }} /> + <div className="text-[10px] uppercase tracking-wide text-text-dimmed">{s}px</div> + </div> + ))} + </div> + </section> + <section className="flex flex-col gap-3"> + <Header2>Variant — white particles</Header2> + <div className="flex flex-wrap items-end gap-10 rounded-md border border-grid-bright bg-background-bright px-6 py-6"> + {[20, 32, 48, 72].map((s) => ( + <div key={s} className="flex flex-col items-center gap-2"> + <TriggerableOrb size={s} orb={{ colored: false, adaptive: true }} /> + <div className="text-[10px] uppercase tracking-wide text-text-dimmed">{s}px</div> + </div> + ))} + </div> + </section> + <section className="flex flex-col gap-3"> + <Header2>Morph scrubber</Header2> + <Scrubber sizes={[20, 32, 48, 72, 140]} orb={{ adaptive: true }} /> + </section> + </div> + ); +} + +// --- shared helpers ------------------------------------------------------------- + +function TriggerableOrb({ size, orb }: { size: number; orb?: Partial<AgentOrbProps> }) { + const [active, setActive] = useState(false); + const timeout = useRef<ReturnType<typeof setTimeout>>(); + + useEffect(() => () => clearTimeout(timeout.current), []); + + const trigger = () => { + setActive(true); + clearTimeout(timeout.current); + timeout.current = setTimeout(() => setActive(false), 4000); + }; + + return ( + <button + type="button" + onClick={trigger} + aria-label="Trigger the agent thinking animation" + title="Click to think" + className="focus-custom cursor-pointer rounded-md p-2 transition hover:bg-background-hover" + > + <AgentOrb size={size} active={active} {...orb} /> + </button> + ); +} + +function Scrubber({ + size = 140, + sizes, + orb, +}: { + size?: number; + sizes?: number[]; + orb?: Partial<AgentOrbProps>; +}) { + const [value, setValue] = useState(0); + const sizeList = sizes ?? [size]; + + return ( + <div className="flex flex-col items-start gap-4 rounded-md border border-grid-bright bg-background-bright px-6 py-6"> + <div className="flex flex-wrap items-end gap-8"> + {sizeList.map((s) => ( + <div key={s} className="flex flex-col items-center gap-2"> + <AgentOrb size={s} activation={value} {...orb} /> + <div className="text-[10px] uppercase tracking-wide text-text-dimmed">{s}px</div> + </div> + ))} + </div> + <div className="flex w-full max-w-md items-center gap-3"> + <span className="w-10 text-xs tabular-nums text-text-dimmed">{value.toFixed(2)}</span> + <input + type="range" + min={0} + max={1} + step={0.01} + value={value} + onChange={(e) => setValue(parseFloat(e.target.value))} + aria-label="Morph amount" + className="h-1 w-full cursor-pointer appearance-none rounded-full bg-grid-bright accent-blue-500" + /> + </div> + <div className="flex w-full max-w-md justify-between text-[10px] uppercase tracking-wide text-text-dimmed"> + <span>Triangle</span> + <span>Working</span> + </div> + </div> + ); +} + +// The trigger.dev brand triangle, but with a solid `currentColor` fill instead of +// LogoIcon's green→yellow gradient (so it can be coloured via a text-* class). +function TriggerTriangle({ className }: { className?: string }) { + return ( + <svg + viewBox="0 0 321 282" + fill="none" + xmlns="http://www.w3.org/2000/svg" + className={className} + aria-hidden + > + <path + fillRule="evenodd" + clipRule="evenodd" + d="M96.1017 113.4L160.679 4.57764e-05L320.718 281.045H0.638916L65.2159 167.642L110.896 194.382L92.0035 227.561H229.354L160.679 106.965L141.786 140.144L96.1017 113.4Z" + fill="currentColor" + /> + </svg> + ); +} + +// ============================================================================ +// Orbit-dots / logo-morph experiment (inlined: contained to this storybook page) +// ============================================================================ + +// Phase 1 prototype of the trigger.dev "AI dashboard agent" orb. +// +// Rest state = a dotted logo (a triangle outline, or the trigger.dev mark filled +// with a dot cloud). Active state = the "working" animation deconstructed from +// thinking-orbs: dots on tilted 3D orbital rings, each with faint "ghost" dots +// tracing the ring plus a few bright particles orbiting it. +// +// A single eased `activation` (0 = rest, 1 = working) interpolates EVERY dot's +// position, radius, depth, alpha and colour between the two layouts — one shared +// pool of dots morphing, not a cross-fade. Colour ramps linearly white -> palette. + +// --- tunables ------------------------------------------------------------- + +const TRIANGLE_SPREAD = 1.45; // scales the triangle to fill the box +const SHELL = 0.82; // orbit shell radius as a fraction of size/2 +const TILT = 0.3; // fixed X-axis tilt of the orbit system (radians) +const SPIN_RATE = 0.12; // how fast the whole system rotates about Y, per time unit + +// Tailwind-500 palette (blue, rose, green, amber). +const AGENT_ORB_PALETTE = ["#3b82f6", "#f43f5e", "#22c55e", "#f59e0b"]; + +// The trigger.dev logo path (from LogoIcon), used to shape the dense dot cloud. +const LOGO_PATH = + "M96.1017 113.4L160.679 4.57764e-05L320.718 281.045H0.638916L65.2159 167.642L110.896 194.382L92.0035 227.561H229.354L160.679 106.965L141.786 140.144L96.1017 113.4Z"; +const LOGO_W = 321; +const LOGO_H = 282; + +// --- pure geometry -------------------------------------------------------- + +type Vec3 = [number, number, number]; +type Pt = [number, number]; + +// Deterministic hash (same as the library's) so orbit orientations are stable. +function hash(i: number, seed: number): number { + const s = Math.sin(i * 12.9898 + seed * 78.233) * 43758.5453; + return s - Math.floor(s); +} + +type OrbitGeom = { + basisA: Vec3; // two orthonormal in-plane vectors spanning the ring + basisB: Vec3; + radiusFactor: number; // ring radius as a fraction of the shell + speed: number; // particle angular speed + direction + phase: number; // shared angular offset for this ring's particles +}; + +function buildOrbitGeoms(orbitCount: number): OrbitGeom[] { + return Array.from({ length: orbitCount }, (_, b) => { + const x = hash(b, 1.7); + const i = hash(b, 5.2); + const p = hash(b, 8.9); + const azimuth = x * 2 * Math.PI; + const polar = Math.acos(2 * i - 1); + const nx = Math.sin(polar) * Math.cos(azimuth); + const ny = Math.cos(polar); + const nz = Math.sin(polar) * Math.sin(azimuth); + let ax = -ny; + let ay = nx; + const len = Math.max(1e-6, Math.hypot(ax, ay)); + ax /= len; + ay /= len; + const basisA: Vec3 = [ax, ay, 0]; + const basisB: Vec3 = [-nz * ay, nz * ax, nx * ay - ny * ax]; // normal × basisA + return { + basisA, + basisB, + radiusFactor: 0.45 + 0.52 * x, + speed: (0.25 + 0.55 * p) * (p > 0.5 ? 1 : -1), + phase: i * 6, + }; + }); +} + +type DotSpec = { + orbit: number; + ghost: boolean; + ringAngle: number; +}; + +// Assign each dot to an orbit (round-robin so rings stay balanced at any count); +// the first `particlesPerOrbit` dots of each ring are bright particles, the rest +// are faint ghosts. Angles are spread evenly around each ring. +function buildDotSpecs(dotCount: number, orbitCount: number, particlesPerOrbit: number): DotSpec[] { + const members: number[][] = Array.from({ length: orbitCount }, () => []); + for (let i = 0; i < dotCount; i++) { + members[i % orbitCount].push(i); + } + const specs: DotSpec[] = new Array(dotCount); + for (let b = 0; b < orbitCount; b++) { + const ring = members[b]; + for (let k = 0; k < ring.length; k++) { + specs[ring[k]] = { + orbit: b, + ghost: k >= particlesPerOrbit, + ringAngle: (k / Math.max(1, ring.length)) * 2 * Math.PI, + }; + } + } + return specs; +} + +// Arc-length parameterizer for a closed polygon: f in [0,1] -> point spaced by +// perimeter distance so dots are evenly distributed. +function makePolygonSampler(points: Pt[]): (f: number) => Pt { + const seg: number[] = []; + let total = 0; + for (let i = 0; i < points.length; i++) { + const a = points[i]; + const b = points[(i + 1) % points.length]; + const d = Math.hypot(b[0] - a[0], b[1] - a[1]); + seg.push(d); + total += d; + } + return (f) => { + let dist = f * total; + let i = 0; + while (dist > seg[i] && i < seg.length - 1) { + dist -= seg[i]; + i++; + } + const a = points[i]; + const b = points[(i + 1) % points.length]; + const t = seg[i] ? Math.min(1, dist / seg[i]) : 0; + return [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t]; + }; +} + +const triangleAt = makePolygonSampler([ + [0, -0.26], + [0.24, 0.16], + [-0.24, 0.16], +]); + +// Rest positions in "fraction of size" units (render maps center + pt * size). +function triangleOutline(count: number): Pt[] { + const pts: Pt[] = []; + for (let i = 0; i < count; i++) { + const [tx, ty] = triangleAt(i / count); + pts.push([tx * TRIANGLE_SPREAD, ty * TRIANGLE_SPREAD]); + } + return pts; +} + +// Project a 3D ring point to screen space (+ depth). Same math as the library's `q`. +function project(x: number, y: number, z: number, spin: number, center: number): Vec3 { + const st = Math.sin(TILT); + const ct = Math.cos(TILT); + const ss = Math.sin(spin); + const cs = Math.cos(spin); + const u = x * cs + z * ss; + const h = -x * ss + z * cs; + const b = y * ct - h * st; + const depth = y * st + h * ct; + return [center + u, center - b, depth]; +} + +function hexToRgb(hex: string): [number, number, number] { + let h = hex.replace("#", ""); + if (h.length === 3) { + h = h[0] + h[0] + h[1] + h[1] + h[2] + h[2]; + } + const n = parseInt(h, 16); + return [(n >> 16) & 255, (n >> 8) & 255, n & 255]; +} + +const mix = (a: number, b: number, t: number) => a + (b - a) * t; + +// Size-adaptive dots: small orbs get fewer, chunkier dots so they stay legible; +// larger orbs keep the denser, finer look. (48px+ matches the non-adaptive 12.) +function adaptiveDots(size: number): { + dotCount: number; + particlesPerOrbit: number; + radiusScale: number; +} { + if (size <= 24) return { dotCount: 6, particlesPerOrbit: 2, radiusScale: 1.7 }; + if (size <= 40) return { dotCount: 9, particlesPerOrbit: 3, radiusScale: 1.25 }; + return { dotCount: 12, particlesPerOrbit: 3, radiusScale: 1 }; +} + +// --- the per-frame painter ------------------------------------------------ + +type RenderConfig = { + size: number; + restPoints: Pt[]; + dotSpecs: DotSpec[]; + orbitGeoms: OrbitGeom[]; + paletteRgb: [number, number, number][]; + restRgb: [number, number, number]; + colored: boolean; + radiusScale: number; +}; + +function drawFrame( + ctx: CanvasRenderingContext2D, + dpr: number, + activation: number, + time: number, + cfg: RenderConfig +) { + const { size, restPoints, dotSpecs, orbitGeoms, paletteRgb, restRgb, colored, radiusScale } = cfg; + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + ctx.clearRect(0, 0, size, size); + + const center = size / 2; + const shell = (size / 2) * SHELL; + const spin = time * SPIN_RATE; + const triRadius = Math.max(0.55, size * 0.03) * radiusScale; + + const dots: { x: number; y: number; r: number; z: number; fill: string }[] = []; + + for (let i = 0; i < dotSpecs.length; i++) { + const home = restPoints[i]; + if (!home) continue; + const spec = dotSpecs[i]; + const geom = orbitGeoms[spec.orbit]; + + const homeX = center + home[0] * size; + const homeY = center + home[1] * size; + + // Live orbit point (3D -> screen). + const ringR = shell * geom.radiusFactor; + const angle = spec.ghost ? spec.ringAngle : time * geom.speed + spec.ringAngle + geom.phase; + const ca = Math.cos(angle); + const sa = Math.sin(angle); + const px = (geom.basisA[0] * ca + geom.basisB[0] * sa) * ringR; + const py = (geom.basisA[1] * ca + geom.basisB[1] * sa) * ringR; + const pz = (geom.basisA[2] * ca + geom.basisB[2] * sa) * ringR; + const [sx, sy, depth] = project(px, py, pz, spin, center); + const front = Math.min(1, Math.max(0, (depth / ringR + 1) / 2)); // 0 back .. 1 front + + const orbitRadius = spec.ghost + ? Math.max(0.4, size * 0.022) * radiusScale + : Math.max(0.5, size * (0.026 + 0.03 * front)) * radiusScale; + const orbitAlpha = spec.ghost ? 0.5 * (0.35 + 0.65 * front) : 0.6 + 0.4 * front; + + // Interpolate rest -> orbit by activation. + const x = mix(homeX, sx, activation); + const y = mix(homeY, sy, activation); + const r = mix(triRadius, orbitRadius, activation); + const alpha = mix(1, orbitAlpha, activation); + + let cr = restRgb[0]; + let cg = restRgb[1]; + let cb = restRgb[2]; + if (colored) { + const target = paletteRgb[i % paletteRgb.length]; + cr = Math.round(mix(restRgb[0], target[0], activation)); + cg = Math.round(mix(restRgb[1], target[1], activation)); + cb = Math.round(mix(restRgb[2], target[2], activation)); + } + + dots.push({ x, y, r, z: depth * activation, fill: `rgba(${cr},${cg},${cb},${alpha})` }); + } + + dots.sort((a, b) => a.z - b.z); // far dots first (no-op at rest since z = depth * 0) + for (const d of dots) { + ctx.fillStyle = d.fill; + ctx.beginPath(); + ctx.arc(d.x, d.y, Math.max(0.3, d.r), 0, Math.PI * 2); + ctx.fill(); + } +} + +// --- component ------------------------------------------------------------ + +type AgentOrbProps = { + /** Rendered size in CSS pixels. Tuned to look right down to ~20px. */ + size?: number; + /** Drives the morph: false = static rest shape, true = working animation. */ + active?: boolean; + /** Controlled override of the morph amount (0..1); ignores `active` when set. */ + activation?: number; + /** Per-dot colours shown while working (cycled by index). */ + colors?: string[]; + /** Ink colour at rest, and for every dot when `colored` is false. */ + restColor?: string; + /** When false, dots stay `restColor` (white) — only alpha/size/position animate. */ + colored?: boolean; + /** Rest layout: a triangle outline, or a dense fill of the trigger.dev logo. */ + restShape?: "triangle"; + /** Total dots in the shared pool. More dots + `logo` shape = solid-logo look. */ + dotCount?: number; + /** Number of orbital rings in the working state. */ + orbitCount?: number; + /** Bright orbiting particles per ring (the rest are faint ghosts). */ + particlesPerOrbit?: number; + /** Auto-tune dot count + dot size to `size` (fewer, chunkier dots when small). */ + adaptive?: boolean; + /** Orbit speed multiplier (3.9 matches the library's 20px "working"). */ + speed?: number; + /** Morph duration when `active` toggles, in ms. */ + transitionMs?: number; + className?: string; + style?: CSSProperties; + "aria-label"?: string; +}; + +function AgentOrb({ + size = 20, + active = false, + activation, + colors = AGENT_ORB_PALETTE, + restColor = "#ffffff", + colored = true, + restShape = "triangle", + dotCount = 21, + orbitCount = 3, + particlesPerOrbit = 3, + adaptive = false, + speed = 3.9, + transitionMs = 600, + className, + style, + "aria-label": ariaLabel, +}: AgentOrbProps) { + const canvasRef = useRef<HTMLCanvasElement>(null); + + const activeRef = useRef(active); + const controlledRef = useRef(activation); + const amountRef = useRef(activation ?? (active ? 1 : 0)); + const timeRef = useRef(0); + const wakeRef = useRef<() => void>(() => {}); + + const colorsKey = colors.join(","); + const adaptiveParams = adaptive ? adaptiveDots(size) : null; + const effDotCount = adaptiveParams?.dotCount ?? dotCount; + const effParticles = adaptiveParams?.particlesPerOrbit ?? particlesPerOrbit; + const radiusScale = adaptiveParams?.radiusScale ?? 1; + const orbitGeoms = useMemo(() => buildOrbitGeoms(orbitCount), [orbitCount]); + const dotSpecs = useMemo( + () => buildDotSpecs(effDotCount, orbitCount, effParticles), + [effDotCount, orbitCount, effParticles] + ); + const restPoints = useMemo(() => triangleOutline(effDotCount), [restShape, effDotCount]); + + useEffect(() => { + activeRef.current = active; + controlledRef.current = activation; + wakeRef.current(); + }, [active, activation]); + + useEffect(() => { + const canvas = canvasRef.current; + const ctx = canvas?.getContext("2d"); + if (!canvas || !ctx) return; + + const dpr = Math.min(2, typeof devicePixelRatio !== "undefined" ? devicePixelRatio : 1); + canvas.width = Math.round(size * dpr); + canvas.height = Math.round(size * dpr); + + const cfg: RenderConfig = { + size, + restPoints, + dotSpecs, + orbitGeoms, + paletteRgb: colors.map(hexToRgb), + restRgb: hexToRgb(restColor), + colored, + radiusScale, + }; + + const reduced = + typeof matchMedia !== "undefined" && matchMedia("(prefers-reduced-motion: reduce)").matches; + + if (reduced) { + // No animation, but a controlled amount (the scrubber) is a static pose, + // so still honour it and repaint when it changes. + const paint = () => { + const amount = controlledRef.current ?? (activeRef.current ? 1 : 0); + amountRef.current = amount; + drawFrame(ctx, dpr, amount, 0, cfg); + }; + wakeRef.current = paint; + paint(); + return () => { + wakeRef.current = () => {}; + }; + } + + let raf = 0; + let running = false; + let last = 0; + + const frame = (now: number) => { + const dt = Math.min(0.05, (now - last) / 1000); + last = now; + + const controlled = controlledRef.current; + let keepGoing: boolean; + + if (controlled != null) { + amountRef.current = controlled; + keepGoing = controlled > 0; + } else { + const target = activeRef.current ? 1 : 0; + const step = transitionMs > 0 ? (dt * 1000) / transitionMs : 1; + if (amountRef.current < target) { + amountRef.current = Math.min(target, amountRef.current + step); + } else if (amountRef.current > target) { + amountRef.current = Math.max(target, amountRef.current - step); + } + keepGoing = amountRef.current !== target || target === 1; + } + + if (amountRef.current > 0) { + timeRef.current += dt * speed; + } + drawFrame(ctx, dpr, amountRef.current, timeRef.current, cfg); + + if (keepGoing) { + raf = requestAnimationFrame(frame); + } else { + running = false; + } + }; + + const wake = () => { + if (running) return; + running = true; + last = typeof performance !== "undefined" ? performance.now() : 0; + raf = requestAnimationFrame(frame); + }; + wakeRef.current = wake; + + drawFrame(ctx, dpr, amountRef.current, timeRef.current, cfg); + wake(); + + return () => { + running = false; + wakeRef.current = () => {}; + cancelAnimationFrame(raf); + }; + }, [ + size, + speed, + transitionMs, + colorsKey, + restColor, + colored, + radiusScale, + orbitGeoms, + dotSpecs, + restPoints, + ]); + + return ( + <canvas + ref={canvasRef} + role="img" + aria-label={ariaLabel ?? (active ? "Agent thinking" : "Agent")} + className={className} + style={{ width: size, height: size, display: "block", ...style }} + /> + ); +} + +// --- crisp logo -> orbits variant ----------------------------------------- +// +// Rest = the real crisp LogoIcon (a separate SVG layer). The 21 dots are +// anchored to the logo's actual outline and born in its green, so as the solid +// mark fades they materialise on its edge, then fly out to the orbits and bloom +// into the palette. Two timing windows make the handoff seamless: +// 0.00 - 0.30 logo fades out; dots fade in, held still on the outline (green) +// 0.30 - 1.00 dots fly outline -> orbit, bloom green -> palette, gain depth + +const smoothstep = (edge0: number, edge1: number, x: number) => { + const t = Math.min(1, Math.max(0, (x - edge0) / (edge1 - edge0))); + return t * t * (3 - 2 * t); +}; + +// 21 points evenly along the logo outline via the real vector path, normalized +// to the same box the crisp LogoIcon fills (width-limited, letterboxed). Cached +// per count; client-only (needs an SVG in the DOM to measure). +const LOGO_OUTLINE_CACHE = new Map<number, Pt[]>(); +function logoOutlinePoints(count: number): Pt[] { + if (typeof document === "undefined") return []; + const cached = LOGO_OUTLINE_CACHE.get(count); + if (cached) return cached; + + const ns = "http://www.w3.org/2000/svg"; + const svg = document.createElementNS(ns, "svg"); + svg.setAttribute("style", "position:absolute;left:-9999px;width:0;height:0;overflow:hidden"); + const path = document.createElementNS(ns, "path") as SVGPathElement; + path.setAttribute("d", LOGO_PATH); + svg.appendChild(path); + document.body.appendChild(svg); + + const total = path.getTotalLength(); + const yOffset = (1 - LOGO_H / LOGO_W) / 2; // LogoIcon is width-limited, so letterboxed vertically + const pts: Pt[] = []; + for (let i = 0; i < count; i++) { + const p = path.getPointAtLength((i / count) * total); + pts.push([p.x / LOGO_W, p.y / LOGO_W + yOffset]); + } + document.body.removeChild(svg); + LOGO_OUTLINE_CACHE.set(count, pts); + return pts; +} + +type LogoMorphConfig = { + size: number; + outline: Pt[]; + dotSpecs: DotSpec[]; + orbitGeoms: OrbitGeom[]; + paletteRgb: [number, number, number][]; + logoRgb: [number, number, number]; +}; + +function drawLogoMorphFrame( + ctx: CanvasRenderingContext2D, + dpr: number, + activation: number, + time: number, + cfg: LogoMorphConfig +) { + const { size, outline, dotSpecs, orbitGeoms, paletteRgb, logoRgb } = cfg; + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + ctx.clearRect(0, 0, size, size); + + const dotOpacity = smoothstep(0.05, 0.3, activation); + if (dotOpacity <= 0.001) return; // rest: only the crisp logo shows + + const fly = smoothstep(0.3, 1, activation); // outline -> orbit + const colorAmount = smoothstep(0.35, 1, activation); // green -> palette + const center = size / 2; + const shell = (size / 2) * SHELL; + const spin = time * SPIN_RATE; + const restRadius = Math.max(0.55, size * 0.03); + + const dots: { x: number; y: number; r: number; z: number; fill: string }[] = []; + for (let i = 0; i < dotSpecs.length; i++) { + const home = outline[i]; + if (!home) continue; + const spec = dotSpecs[i]; + const geom = orbitGeoms[spec.orbit]; + + const homeX = home[0] * size; // absolute fraction, aligned to the crisp logo + const homeY = home[1] * size; + + const ringR = shell * geom.radiusFactor; + const angle = spec.ghost ? spec.ringAngle : time * geom.speed + spec.ringAngle + geom.phase; + const ca = Math.cos(angle); + const sa = Math.sin(angle); + const px = (geom.basisA[0] * ca + geom.basisB[0] * sa) * ringR; + const py = (geom.basisA[1] * ca + geom.basisB[1] * sa) * ringR; + const pz = (geom.basisA[2] * ca + geom.basisB[2] * sa) * ringR; + const [sx, sy, depth] = project(px, py, pz, spin, center); + const front = Math.min(1, Math.max(0, (depth / ringR + 1) / 2)); + + const orbitRadius = spec.ghost + ? Math.max(0.4, size * 0.022) + : Math.max(0.5, size * (0.026 + 0.03 * front)); + const orbitAlpha = spec.ghost ? 0.5 * (0.35 + 0.65 * front) : 0.6 + 0.4 * front; + + const x = mix(homeX, sx, fly); + const y = mix(homeY, sy, fly); + const r = mix(restRadius, orbitRadius, fly); + const alpha = mix(1, orbitAlpha, fly) * dotOpacity; + const target = paletteRgb[i % paletteRgb.length]; + const cr = Math.round(mix(logoRgb[0], target[0], colorAmount)); + const cg = Math.round(mix(logoRgb[1], target[1], colorAmount)); + const cb = Math.round(mix(logoRgb[2], target[2], colorAmount)); + + dots.push({ x, y, r, z: depth * fly, fill: `rgba(${cr},${cg},${cb},${alpha})` }); + } + + dots.sort((a, b) => a.z - b.z); + for (const d of dots) { + ctx.fillStyle = d.fill; + ctx.beginPath(); + ctx.arc(d.x, d.y, Math.max(0.3, d.r), 0, Math.PI * 2); + ctx.fill(); + } +} + +type AgentLogoMorphProps = { + size?: number; + active?: boolean; + activation?: number; + colors?: string[]; + /** Colour the dots are born as during the handoff (matches the logo green). */ + logoColor?: string; + dotCount?: number; + orbitCount?: number; + particlesPerOrbit?: number; + speed?: number; + transitionMs?: number; + className?: string; + style?: CSSProperties; + "aria-label"?: string; +}; + +function AgentLogoMorph({ + size = 20, + active = false, + activation, + colors = AGENT_ORB_PALETTE, + logoColor = "#41ff54", + dotCount = 21, + orbitCount = 3, + particlesPerOrbit = 3, + speed = 3.9, + transitionMs = 600, + className, + style, + "aria-label": ariaLabel, +}: AgentLogoMorphProps) { + const canvasRef = useRef<HTMLCanvasElement>(null); + const logoRef = useRef<HTMLDivElement>(null); + + const activeRef = useRef(active); + const controlledRef = useRef(activation); + const amountRef = useRef(activation ?? (active ? 1 : 0)); + const timeRef = useRef(0); + const wakeRef = useRef<() => void>(() => {}); + + const colorsKey = colors.join(","); + + useEffect(() => { + activeRef.current = active; + controlledRef.current = activation; + wakeRef.current(); + }, [active, activation]); + + useEffect(() => { + const canvas = canvasRef.current; + const ctx = canvas?.getContext("2d"); + if (!canvas || !ctx) return; + + const dpr = Math.min(2, typeof devicePixelRatio !== "undefined" ? devicePixelRatio : 1); + canvas.width = Math.round(size * dpr); + canvas.height = Math.round(size * dpr); + + const cfg: LogoMorphConfig = { + size, + outline: logoOutlinePoints(dotCount), + dotSpecs: buildDotSpecs(dotCount, orbitCount, particlesPerOrbit), + orbitGeoms: buildOrbitGeoms(orbitCount), + paletteRgb: colors.map(hexToRgb), + logoRgb: hexToRgb(logoColor), + }; + + const setLogo = (a: number) => { + const el = logoRef.current; + if (!el) return; + const hand = smoothstep(0, 0.3, a); + el.style.opacity = String(1 - hand); + el.style.transform = `scale(${1 - 0.06 * hand})`; + el.style.visibility = hand >= 0.999 ? "hidden" : "visible"; + }; + + const reduced = + typeof matchMedia !== "undefined" && matchMedia("(prefers-reduced-motion: reduce)").matches; + if (reduced) { + // Same as the orb: freeze the animation, but still render a controlled + // amount so the scrubber keeps working. + const paint = () => { + const amount = controlledRef.current ?? (activeRef.current ? 1 : 0); + amountRef.current = amount; + setLogo(amount); + drawLogoMorphFrame(ctx, dpr, amount, 0, cfg); + }; + wakeRef.current = paint; + paint(); + return () => { + wakeRef.current = () => {}; + }; + } + + let raf = 0; + let running = false; + let last = 0; + + const frame = (now: number) => { + const dt = Math.min(0.05, (now - last) / 1000); + last = now; + + const controlled = controlledRef.current; + let keepGoing: boolean; + if (controlled != null) { + amountRef.current = controlled; + keepGoing = controlled > 0; + } else { + const target = activeRef.current ? 1 : 0; + const step = transitionMs > 0 ? (dt * 1000) / transitionMs : 1; + if (amountRef.current < target) { + amountRef.current = Math.min(target, amountRef.current + step); + } else if (amountRef.current > target) { + amountRef.current = Math.max(target, amountRef.current - step); + } + keepGoing = amountRef.current !== target || target === 1; + } + + const a = amountRef.current; + if (a > 0) { + timeRef.current += dt * speed; + } + setLogo(a); + drawLogoMorphFrame(ctx, dpr, a, timeRef.current, cfg); + + if (keepGoing) { + raf = requestAnimationFrame(frame); + } else { + running = false; + } + }; + + const wake = () => { + if (running) return; + running = true; + last = typeof performance !== "undefined" ? performance.now() : 0; + raf = requestAnimationFrame(frame); + }; + wakeRef.current = wake; + + setLogo(amountRef.current); + drawLogoMorphFrame(ctx, dpr, amountRef.current, timeRef.current, cfg); + wake(); + + return () => { + running = false; + wakeRef.current = () => {}; + cancelAnimationFrame(raf); + }; + }, [size, speed, transitionMs, colorsKey, logoColor, dotCount, orbitCount, particlesPerOrbit]); + + return ( + <span + className={className} + style={{ position: "relative", display: "inline-block", width: size, height: size, ...style }} + role="img" + aria-label={ariaLabel ?? (active ? "Agent thinking" : "Agent")} + > + <div + ref={logoRef} + style={{ + position: "absolute", + inset: 0, + transformOrigin: "center", + willChange: "opacity, transform", + }} + > + <LogoIcon className="h-full w-full" /> + </div> + <canvas + ref={canvasRef} + aria-hidden + style={{ position: "absolute", inset: 0, width: "100%", height: "100%" }} + /> + </span> + ); +} diff --git a/apps/webapp/app/routes/storybook.buttons/route.tsx b/apps/webapp/app/routes/storybook.buttons/route.tsx index 702bcbfa976..17d3a7280d7 100644 --- a/apps/webapp/app/routes/storybook.buttons/route.tsx +++ b/apps/webapp/app/routes/storybook.buttons/route.tsx @@ -399,6 +399,26 @@ export default function Story() { </Button> </div> </div> + <Header1 className="mb-2 mt-8">Ask AI button</Header1> + <div className="grid grid-cols-4 gap-8 border-b border-grid-bright pb-8"> + <div className="flex flex-col items-start gap-2"> + <Header3 className="mb-1 uppercase">Sizes</Header3> + <Button variant="ask-ai/small">Ask AI</Button> + <Button variant="ask-ai/medium">Ask AI</Button> + <Button variant="ask-ai/large">Ask AI</Button> + </div> + <div className="flex flex-col items-start gap-2"> + <Header3 className="mb-1 uppercase">Disabled</Header3> + <Button variant="ask-ai/small" disabled> + Ask AI + </Button> + </div> + <div className="col-span-2 self-end text-sm text-text-dimmed"> + Reserved for the AI agent. The variant supplies the agent logo automatically, so no + LeadingIcon is needed. Pass one to animate it. See the "AI agent" story for the live + pairing. + </div> + </div> </div> ); } diff --git a/apps/webapp/app/routes/storybook/route.tsx b/apps/webapp/app/routes/storybook/route.tsx index 8309fee5b2b..e033f5d8ee2 100644 --- a/apps/webapp/app/routes/storybook/route.tsx +++ b/apps/webapp/app/routes/storybook/route.tsx @@ -7,6 +7,10 @@ import { requireUser } from "~/services/session.server"; import { cn } from "~/utils/cn"; const stories: Story[] = [ + { + name: "AI agent", + slug: "ai-agent", + }, { name: "Animated panel", slug: "animated-panel", From 0445b8ec27f0efd37adc577c539b934a03224c20 Mon Sep 17 00:00:00 2001 From: Eric Allam <eric@trigger.dev> Date: Sat, 1 Aug 2026 09:17:20 +0100 Subject: [PATCH 52/67] fix(webapp,clickhouse): keep the rest of a ClickHouse batch when one run or span has un-ingestable JSON (#4358) ## Summary A single run output, trace span, or payload carrying JSON that ClickHouse can't ingest (for example nesting past its depth limit) used to fail the whole insert batch, so unrelated runs and spans silently disappeared from the runs list, traces, and logs. This keeps the rest of the batch and handles the offending row instead of dropping everything around it. ## Fix Recovery is per-table, matched to what each table needs: - **Runs** (`task_runs_v2`) keep their status. We follow ClickHouse's failing-row hint to strip just the un-ingestable JSON column(s) so the run still lands (its output reads from Postgres on the detail page), up to a configurable limit (`RUN_REPLICATION_MAX_POISON_STRIPS_PER_BATCH`, default `1`). Past the limit we stop and land the batch with `allow_errors` in a single pass, skipping the remainder. Cost stays a fixed handful of inserts no matter how large or poisoned a flush is. - **Trace events and payloads** (high volume, append-only) recover with a single `allow_errors` insert: the good rows land in one pass and only the un-ingestable rows are skipped. Before falling back, a lightweight sanitizer still repairs what it can losslessly (lone UTF-16 surrogates, out-of-range integers) so a repairable row lands in full. To read the failing-row hint we patch `@clickhouse/client-common`: its error parser truncates the server response and discards the `(at row N)` position, so the patch preserves the full text for the recovery path to read. --- ...uns-list-batch-drop-on-unparseable-json.md | 6 + apps/webapp/app/env.server.ts | 1 + .../runsReplicationInstance.server.ts | 1 + .../services/runsReplicationService.server.ts | 333 +++++---- .../clickhouseEventRepository.server.ts | 227 +++--- .../sanitizeRowsOnParseError.server.ts | 471 ++++++++++++ ...ckhouseEventRepositoryJsonRecovery.test.ts | 125 ++++ .../test/runsReplicationBenchmark.producer.ts | 24 + ...nsReplicationJsonRecoveryBenchmark.test.ts | 287 ++++++++ .../runsReplicationService.part10.test.ts | 230 ++++++ .../test/sanitizeRowsOnParseError.test.ts | 678 ++++++++++++++++++ .../clickhouse/src/client/client.ts | 13 +- .../clickhouse/src/client/errors.test.ts | 47 ++ .../clickhouse/src/client/errors.ts | 14 +- package.json | 3 +- .../@clickhouse__client-common@1.12.1.patch | 29 + pnpm-lock.yaml | 7 +- 17 files changed, 2237 insertions(+), 259 deletions(-) create mode 100644 .server-changes/fix-runs-list-batch-drop-on-unparseable-json.md create mode 100644 apps/webapp/test/clickhouseEventRepositoryJsonRecovery.test.ts create mode 100644 apps/webapp/test/runsReplicationJsonRecoveryBenchmark.test.ts create mode 100644 apps/webapp/test/runsReplicationService.part10.test.ts create mode 100644 internal-packages/clickhouse/src/client/errors.test.ts create mode 100644 patches/@clickhouse__client-common@1.12.1.patch diff --git a/.server-changes/fix-runs-list-batch-drop-on-unparseable-json.md b/.server-changes/fix-runs-list-batch-drop-on-unparseable-json.md new file mode 100644 index 00000000000..fe56e609510 --- /dev/null +++ b/.server-changes/fix-runs-list-batch-drop-on-unparseable-json.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Fixed a rare case where a single run or span carrying data that could not be ingested would make other runs or trace events in the same batch go missing from the runs list, traces, and logs. Now the rest of the batch is always kept: an affected run still appears with its status (only its un-ingestable output is dropped), and an affected trace event or payload is skipped instead of taking down everything around it. diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index b371d5994e9..9e0a9fa84ed 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1723,6 +1723,7 @@ const EnvironmentSchema = z RUN_REPLICATION_MAX_FLUSH_CONCURRENCY: z.coerce.number().int().default(2), RUN_REPLICATION_FLUSH_INTERVAL_MS: z.coerce.number().int().default(1000), RUN_REPLICATION_FLUSH_BATCH_SIZE: z.coerce.number().int().default(100), + RUN_REPLICATION_MAX_POISON_STRIPS_PER_BATCH: z.coerce.number().int().default(1), RUN_REPLICATION_LEADER_LOCK_TIMEOUT_MS: z.coerce.number().int().default(30_000), RUN_REPLICATION_LEADER_LOCK_EXTEND_INTERVAL_MS: z.coerce.number().int().default(10_000), RUN_REPLICATION_ACK_INTERVAL_SECONDS: z.coerce.number().int().default(10), diff --git a/apps/webapp/app/services/runsReplicationInstance.server.ts b/apps/webapp/app/services/runsReplicationInstance.server.ts index 15950aa9008..164ce07fb92 100644 --- a/apps/webapp/app/services/runsReplicationInstance.server.ts +++ b/apps/webapp/app/services/runsReplicationInstance.server.ts @@ -117,6 +117,7 @@ function initializeRunsReplicationInstance() { maxFlushConcurrency: env.RUN_REPLICATION_MAX_FLUSH_CONCURRENCY, flushIntervalMs: env.RUN_REPLICATION_FLUSH_INTERVAL_MS, flushBatchSize: env.RUN_REPLICATION_FLUSH_BATCH_SIZE, + maxPoisonStripsPerBatch: env.RUN_REPLICATION_MAX_POISON_STRIPS_PER_BATCH, leaderLockTimeoutMs: env.RUN_REPLICATION_LEADER_LOCK_TIMEOUT_MS, leaderLockExtendIntervalMs: env.RUN_REPLICATION_LEADER_LOCK_EXTEND_INTERVAL_MS, leaderLockAcquireAdditionalTimeMs: env.RUN_REPLICATION_LEADER_LOCK_ADDITIONAL_TIME_MS, diff --git a/apps/webapp/app/services/runsReplicationService.server.ts b/apps/webapp/app/services/runsReplicationService.server.ts index 732665461dd..4ddeb2af171 100644 --- a/apps/webapp/app/services/runsReplicationService.server.ts +++ b/apps/webapp/app/services/runsReplicationService.server.ts @@ -1,11 +1,13 @@ import type { ClickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; import { type ClickHouse, + type ClickHouseSettings, type PayloadInsertArray, type TaskRunInsertArray, composeTaskRunVersion, getPayloadField, getTaskRunField, + TASK_RUN_INDEX, } from "@internal/clickhouse"; import { type RedisOptions } from "@internal/redis"; import { @@ -41,9 +43,9 @@ import { detectBadJsonStrings } from "~/utils/detectBadJsonStrings"; import { calculateErrorFingerprint } from "~/utils/errorFingerprinting"; import { baseWorkerQueue } from "~/runEngine/concerns/workerQueueSplit.server"; import { - isClickHouseJsonParseError, - parseRowNumberFromError, - sanitizeRows, + insertWithBadRowSkip, + insertWithLimitedStrip, + type JsonParseRecoveryOutcome, } from "~/v3/eventRepository/sanitizeRowsOnParseError.server"; interface TransactionEvent<T = any> { @@ -112,6 +114,7 @@ export type RunsReplicationServiceOptions = { insertMaxDelayMs?: number; disablePayloadInsert?: boolean; disableErrorFingerprinting?: boolean; + maxPoisonStripsPerBatch?: number; }; type PostgresTaskRun = TaskRun & { masterQueue: string }; @@ -175,14 +178,47 @@ export class RunsReplicationService { private _disableErrorFingerprinting: boolean; /** - * Counts batches that hit a ClickHouse `Cannot parse JSON object` failure - * that survived one sanitize-retry. These batches are dropped on the floor - * (returning success-ish to the caller so the retry layer doesn't spin on - * the same deterministic failure), and we track the drop count for - * observability. Counter only — does not gate behaviour. + * Counts batches where every row was un-ingestable even with its JSON + * stripped, so nothing landed. Row isolation lands anything strippable, so + * this should stay at zero; a non-zero value means the recovery itself is + * failing. Only incremented when ClickHouse's summary says so exactly + * (`written_rows === 0`). Counter only, does not gate behaviour. */ private _permanentlyDroppedBatches = 0; + /** + * Counts batches that took the row-isolation recovery path: a + * `Cannot parse JSON object` failure the sanitizer could not repair, where we + * followed ClickHouse's `at row N` hint to the poison rows and landed the batch + * with their JSON stripped. Reliable per-event signal that user data is hitting + * the ceiling. + */ + private _rowIsolationRecoveries = 0; + + /** + * Counts batches that gave up on isolating rows precisely and fell back to a + * single `allow_errors` insert, either because the per-batch strip budget was + * spent (a poison flood) or because ClickHouse gave no usable row hint. The + * remaining un-ingestable rows are skipped; `bailReason` in the log line says + * which cause it was. + */ + private _recoveryCapHits = 0; + + /** + * Counts rows that landed with their un-ingestable `output` emptied (the run + * kept its status, only the output content was lost; it still reads from + * Postgres on the run detail page). + */ + private _rowsStripped = 0; + + /** + * Counts rows dropped entirely because they could not be parsed even with + * their JSON stripped. The true data-loss signal; expected to stay near zero. + * A lower bound on `task_runs_v2`, whose materialized views make the exact + * count underivable from ClickHouse's insert summary (see `droppedRowCount`). + */ + private _permanentlyDroppedRows = 0; + // Metrics private _replicationLagHistogram: Histogram; private _batchesFlushedCounter: Counter; @@ -191,6 +227,11 @@ export class RunsReplicationService { private _payloadsInsertedCounter: Counter; private _insertRetriesCounter: Counter; private _eventsProcessedCounter: Counter; + private _rowIsolatedBatchesCounter: Counter; + private _recoveryCapHitsCounter: Counter; + private _rowsStrippedCounter: Counter; + private _rowsDroppedCounter: Counter; + private _droppedBatchesCounter: Counter; private _flushDurationHistogram: Histogram; public readonly events: EventEmitter<RunsReplicationServiceEvents>; @@ -244,6 +285,38 @@ export class RunsReplicationService { description: "Replication events processed (inserts, updates, deletes)", }); + this._rowIsolatedBatchesCounter = this._meter.createCounter( + "runs_replication.batches_row_isolated", + { + description: + "Batches recovered by isolating un-ingestable rows (landed the rest) after a ClickHouse JSON parse error", + unit: "batches", + } + ); + + this._recoveryCapHitsCounter = this._meter.createCounter("runs_replication.recovery_cap_hits", { + description: + "Batches that fell back to a single allow_errors insert instead of isolating rows, because the per-batch strip budget was spent or ClickHouse gave no usable row hint", + unit: "batches", + }); + + this._rowsStrippedCounter = this._meter.createCounter("runs_replication.rows_stripped", { + description: + "Rows landed with their un-ingestable JSON stripped (kept the row, lost only the JSON content)", + unit: "rows", + }); + + this._rowsDroppedCounter = this._meter.createCounter("runs_replication.rows_dropped", { + description: + "Rows dropped entirely because they could not parse even with their JSON stripped; a lower bound on task_runs_v2, whose materialized views make the exact count underivable from ClickHouse's insert summary", + unit: "rows", + }); + + this._droppedBatchesCounter = this._meter.createCounter("runs_replication.batches_dropped", { + description: "Batches where every row was un-ingestable even with its JSON stripped", + unit: "batches", + }); + this._flushDurationHistogram = this._meter.createHistogram( "runs_replication.flush_duration_ms", { @@ -411,11 +484,31 @@ export class RunsReplicationService { }); } - /** Exposed for tests and metrics — total batches lost to unrecoverable parse errors. */ + /** Exposed for tests and metrics — batches where nothing landed even after stripping JSON. */ get permanentlyDroppedBatches() { return this._permanentlyDroppedBatches; } + /** Exposed for tests and metrics — batches that took the row-isolation recovery path. */ + get rowIsolationRecoveries() { + return this._rowIsolationRecoveries; + } + + /** Exposed for tests and metrics — batches whose isolation hit the per-batch insert budget. */ + get recoveryCapHits() { + return this._recoveryCapHits; + } + + /** Exposed for tests and metrics — rows that landed with their un-ingestable JSON stripped. */ + get rowsStripped() { + return this._rowsStripped; + } + + /** Exposed for tests and metrics — rows dropped entirely (could not parse even stripped). */ + get permanentlyDroppedRows() { + return this._permanentlyDroppedRows; + } + public async shutdown() { if (this._isShuttingDown) return; @@ -870,15 +963,11 @@ export class RunsReplicationService { payloadError = plErr; } - // Only count rows that actually landed in ClickHouse. `kind: "dropped"` - // means the recovery wrapper bailed (sanitizer no-op or sanitize-retry - // still failed) — those rows never made it, so they must not show up - // as successful inserts in the per-batch counter. - if (!trErr && trOutcome?.kind !== "dropped") { - this._taskRunsInsertedCounter.add(group.taskRunInserts.length); + if (!trErr && trOutcome) { + this._taskRunsInsertedCounter.add(landedRowCount(group.taskRunInserts.length, trOutcome)); } - if (!plErr && plOutcome?.kind !== "dropped") { - this._payloadsInsertedCounter.add(group.payloadInserts.length); + if (!plErr && plOutcome) { + this._payloadsInsertedCounter.add(landedRowCount(group.payloadInserts.length, plOutcome)); } } @@ -1034,11 +1123,12 @@ export class RunsReplicationService { return; } return await startSpan(this._tracer, "insertTaskRunsInserts", async (span) => { - const doInsert = async () => { - const [insertError, insertResult] = await clickhouse.taskRuns.insertCompactArrays( - taskRunInserts, - { params: { clickhouse_settings: this.#getClickhouseInsertSettings() } } - ); + const rawInsert = async (rows: TaskRunInsertArray[], extraSettings?: ClickHouseSettings) => { + const [insertError, insertResult] = await clickhouse.taskRuns.insertCompactArrays(rows, { + params: { + clickhouse_settings: { ...this.#getClickhouseInsertSettings(), ...extraSettings }, + }, + }); if (insertError) { this.logger.error("Error inserting task run inserts attempt", { error: insertError, @@ -1050,12 +1140,27 @@ export class RunsReplicationService { return insertResult; }; - return await this.#insertWithJsonParseRecovery( - taskRunInserts, - doInsert, - "task_runs_v2", - attempt - ); + const outcome = await insertWithLimitedStrip({ + rows: taskRunInserts, + contextLabel: "task_runs_v2", + logger: this.logger, + logContext: { attempt }, + insert: (rows) => rawInsert(rows), + insertSync: (rows) => + rawInsert(rows, { async_insert: 0, input_format_parallel_parsing: 0 }), + insertAllowingBadRows: (rows) => + rawInsert(rows, { + async_insert: 0, + input_format_parallel_parsing: 0, + input_format_allow_errors_num: String(rows.length), + input_format_allow_errors_ratio: 1, + }), + stripJsonColumns: stripTaskRunJsonColumns, + maxPoisonStrips: this.options.maxPoisonStripsPerBatch, + hasMaterializedViews: true, + }); + this.#recordRecoveryOutcome(outcome, "task_runs_v2", taskRunInserts.length); + return outcome; }); } @@ -1068,10 +1173,14 @@ export class RunsReplicationService { return; } return await startSpan(this._tracer, "insertPayloadInserts", async (span) => { - const doInsert = async () => { + const rawInsert = async (rows: PayloadInsertArray[], extraSettings?: ClickHouseSettings) => { const [insertError, insertResult] = await clickhouse.taskRuns.insertPayloadsCompactArrays( - payloadInserts, - { params: { clickhouse_settings: this.#getClickhouseInsertSettings() } } + rows, + { + params: { + clickhouse_settings: { ...this.#getClickhouseInsertSettings(), ...extraSettings }, + }, + } ); if (insertError) { this.logger.error("Error inserting payload inserts attempt", { @@ -1084,111 +1193,54 @@ export class RunsReplicationService { return insertResult; }; - return await this.#insertWithJsonParseRecovery( - payloadInserts, - doInsert, - "raw_task_runs_payload_v1", - attempt - ); + const outcome = await insertWithBadRowSkip({ + rows: payloadInserts, + contextLabel: "raw_task_runs_payload_v1", + logger: this.logger, + logContext: { attempt }, + insert: (rows) => rawInsert(rows), + insertAllowingBadRows: (rows) => + rawInsert(rows, { + async_insert: 0, + input_format_parallel_parsing: 0, + input_format_allow_errors_num: String(rows.length), + input_format_allow_errors_ratio: 1, + }), + hasMaterializedViews: false, + }); + this.#recordRecoveryOutcome(outcome, "raw_task_runs_payload_v1", payloadInserts.length); + return outcome; }); } - /** - * Wraps a ClickHouse insert with reactive UTF-16 sanitization for - * `Cannot parse JSON object` rejections. Mirrors the pattern from - * `ClickhouseEventRepository.#insertWithJsonParseRecovery` introduced - * in #3659 — same root cause (lone UTF-16 surrogates in user-provided - * JSON), same recovery shape: - * - * 1. Try the insert. Healthy batches pay zero scan cost. - * 2. On parse error, walk the whole batch via `sanitizeRows` and - * replace any lone-surrogate string with `"[invalid-utf16]"`. - * 3. Retry once. If the sanitizer found nothing or the retry also - * fails with the same error class, drop the batch loudly and - * return — do NOT rethrow, otherwise the surrounding - * `#insertWithRetry` layer would spin three more times on the - * same deterministic failure. - * 4. Non-parse errors propagate unchanged so the existing - * transient-retry path still handles them. - * - * The whole-batch scan (rather than slicing on the `at row N` hint) is - * deliberate: `at row N` semantics under `input_format_parallel_parsing` - * aren't stable enough to safely skip rows. The cost is bounded because - * `detectBadJsonStrings` exits in O(1) for clean strings. - */ - async #insertWithJsonParseRecovery<T extends object>( - rows: T[], - doInsert: () => Promise<unknown>, + #recordRecoveryOutcome( + outcome: JsonParseRecoveryOutcome, contextLabel: string, - attempt: number - ): Promise< - | { kind: "inserted"; insertResult: unknown } - | { kind: "sanitized"; insertResult: unknown } - | { kind: "dropped" } - > { - try { - return { kind: "inserted", insertResult: await doInsert() }; - } catch (firstError) { - if (!isClickHouseJsonParseError(firstError)) throw firstError; - - const firstMessage = - typeof firstError === "object" && firstError !== null && "message" in firstError - ? String((firstError as { message?: unknown }).message ?? "") - : String(firstError); - - const rowHint = parseRowNumberFromError(firstMessage); - const { rowsTouched, fieldsSanitized } = sanitizeRows(rows); - - if (fieldsSanitized === 0) { - this._permanentlyDroppedBatches += 1; - this.logger.error( - "Dropped batch — ClickHouse JSON parse error but sanitizer found nothing to fix", - { - contextLabel, - attempt, - batchSize: rows.length, - clickhouseRowHint: rowHint, - permanentlyDroppedBatches: this._permanentlyDroppedBatches, - sampleRow: JSON.stringify(rows[0] ?? null).slice(0, 1024), - clickhouseError: firstMessage.split("\n")[0], - } - ); - return { kind: "dropped" }; - } + batchSize: number + ) { + if (outcome.kind !== "recovered") { + return; + } - this.logger.warn("Sanitizing batch after ClickHouse JSON parse error", { - contextLabel, - attempt, - batchSize: rows.length, - clickhouseRowHint: rowHint, - rowsTouched, - fieldsSanitized, - clickhouseError: firstMessage.split("\n")[0], - }); + this._rowIsolationRecoveries += 1; + this._rowIsolatedBatchesCounter.add(1, { table: contextLabel }); - try { - return { kind: "sanitized", insertResult: await doInsert() }; - } catch (retryError) { - if (!isClickHouseJsonParseError(retryError)) throw retryError; + if (outcome.capped) { + this._recoveryCapHits += 1; + this._recoveryCapHitsCounter.add(1, { table: contextLabel }); + } + + if (outcome.rowsStripped > 0) { + this._rowsStripped += outcome.rowsStripped; + this._rowsStrippedCounter.add(outcome.rowsStripped, { table: contextLabel }); + } + if (outcome.rowsDropped > 0) { + this._permanentlyDroppedRows += outcome.rowsDropped; + this._rowsDroppedCounter.add(outcome.rowsDropped, { table: contextLabel }); + if (outcome.rowsDroppedExact && outcome.rowsDropped === batchSize) { this._permanentlyDroppedBatches += 1; - const retryMessage = - typeof retryError === "object" && retryError !== null && "message" in retryError - ? String((retryError as { message?: unknown }).message ?? "") - : String(retryError); - this.logger.error( - "Dropped batch after sanitize-retry still hit ClickHouse JSON parse error", - { - contextLabel, - attempt, - batchSize: rows.length, - permanentlyDroppedBatches: this._permanentlyDroppedBatches, - sampleRow: JSON.stringify(rows[0] ?? null).slice(0, 1024), - firstError: firstMessage.split("\n")[0], - retryError: retryMessage.split("\n")[0], - } - ); - return { kind: "dropped" }; + this._droppedBatchesCounter.add(1, { table: contextLabel }); } } } @@ -1542,3 +1594,28 @@ function lsnToUInt64(lsn: string): bigint { const [seg, off] = lsn.split("/"); return (BigInt("0x" + seg) << 32n) | BigInt("0x" + off); } + +function landedRowCount(groupSize: number, outcome: JsonParseRecoveryOutcome): number { + if (outcome.kind === "recovered") { + return Math.max(0, groupSize - outcome.rowsDropped); + } + return groupSize; +} + +const STRIPPED_JSON: { data: unknown } = { data: undefined }; + +/** + * Empties `output`, the run JSON that in practice exceeds what ClickHouse can + * ingest (a large or deeply nested task return value). `error` is deliberately + * left alone: emptying it while keeping `error_fingerprint` would let the row + * match the error materialized views with no error content, and since those + * views pick their display columns with `any()` over the fingerprint group, one + * stripped run could retitle every run sharing that fingerprint. A run whose + * `error` is itself un-ingestable therefore makes no progress here and falls + * through to the `allow_errors` bail, which skips just that row. + */ +function stripTaskRunJsonColumns(row: TaskRunInsertArray): TaskRunInsertArray { + const stripped = [...row] as TaskRunInsertArray; + stripped[TASK_RUN_INDEX.output] = STRIPPED_JSON; + return stripped; +} diff --git a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts index 573097005ed..ef81942164c 100644 --- a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts +++ b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts @@ -1,5 +1,6 @@ import type { ClickHouse, + ClickHouseSettings, LlmMetricsV1Input, MetricsV1Input, TaskEventDetailedSummaryV1Result, @@ -66,9 +67,9 @@ import type { TraceSummary, } from "./eventRepository.types"; import { - isClickHouseJsonParseError, - parseRowNumberFromError, - sanitizeRows, + insertWithBadRowSkip, + type JsonParseRecoveryOutcome, + landedNothing, } from "./sanitizeRowsOnParseError.server"; export type ClickhouseEventRepositoryConfig = { @@ -121,14 +122,32 @@ export class ClickhouseEventRepository implements IEventRepository { private _tracer: Tracer; private _version: "v1" | "v2"; /** - * Counts batches that hit a ClickHouse JSON parse failure that survived - * one sanitize-retry. These batches are dropped on the floor (the scheduler - * is told the flush "succeeded" so its queue counter doesn't leak), and we - * track the drop count for observability. + * Counts batches where every row was un-ingestable, so nothing landed. Only + * incremented when ClickHouse's summary says so exactly (`written_rows === 0`); + * expected to stay at zero, since a whole batch of un-ingestable events means + * something upstream is broken rather than one bad payload. */ private _permanentlyDroppedBatches = 0; private readonly _droppedBatchesCounter: Counter; + /** + * Counts batches that took the bad-row-skip recovery path: a + * `Cannot parse JSON object` failure the sanitizer could not repair, where one + * `allow_errors` insert landed the good rows and skipped the un-ingestable + * ones. Every such batch lost at least one row, so this is the alertable + * signal for these tables. + */ + private _rowIsolationRecoveries = 0; + private readonly _rowIsolatedBatchesCounter: Counter; + + /** + * Counts rows skipped as un-ingestable. A floor, not an exact count: these + * tables carry row-multiplying materialized views, so ClickHouse's insert + * summary can't separate skipped base rows from MV rows (see `droppedRowCount`). + */ + private _permanentlyDroppedRows = 0; + private readonly _rowsDroppedCounter: Counter; + constructor(config: ClickhouseEventRepositoryConfig) { this._clickhouse = config.clickhouse; this._config = config; @@ -140,6 +159,16 @@ export class ClickhouseEventRepository implements IEventRepository { description: "Batches permanently dropped after an unrecoverable ClickHouse JSON parse error", unit: "batches", }); + this._rowIsolatedBatchesCounter = meter.createCounter("ingest.flush.batches_row_isolated", { + description: + "Batches recovered by skipping un-ingestable rows (landed the rest) after a ClickHouse JSON parse error; each lost at least one row", + unit: "batches", + }); + this._rowsDroppedCounter = meter.createCounter("ingest.flush.rows_dropped", { + description: + "Rows skipped as un-ingestable, as a lower bound: these tables' materialized views make the exact count underivable from ClickHouse's insert summary", + unit: "rows", + }); this._flushScheduler = new DynamicFlushScheduler({ name: `task_events_${this._version}`, @@ -189,11 +218,21 @@ export class ClickhouseEventRepository implements IEventRepository { return this._config.maximumLiveReloadingSetting ?? 1000; } - /** Exposed for tests and metrics — total batches lost to unrecoverable parse errors. */ + /** Exposed for tests and metrics — batches where nothing landed even after stripping JSON. */ get permanentlyDroppedBatches() { return this._permanentlyDroppedBatches; } + /** Exposed for tests and metrics — batches that took the bad-row-skip recovery path. */ + get rowIsolationRecoveries() { + return this._rowIsolationRecoveries; + } + + /** Exposed for tests and metrics — rows skipped as un-ingestable (a lower bound). */ + get permanentlyDroppedRows() { + return this._permanentlyDroppedRows; + } + /** * Clamps a start time (in nanoseconds) to now if it's too far in the past. * Returns the clamped value as a bigint. @@ -262,32 +301,43 @@ export class ClickhouseEventRepository implements IEventRepository { ? this._clickhouse.taskEventsV2.insert : this._clickhouse.taskEvents.insert; - const doInsert = async () => { - const [insertError, insertResult] = await insertFn(events, { + const contextLabel = `task_events_${this._version}`; + const rawInsert = async ( + rows: (TaskEventV1Input | TaskEventV2Input)[], + extraSettings?: ClickHouseSettings + ) => { + const [insertError, insertResult] = await insertFn(rows, { params: { - clickhouse_settings: this.#getClickhouseInsertSettings(), + clickhouse_settings: { ...this.#getClickhouseInsertSettings(), ...extraSettings }, }, }); if (insertError) throw insertError; return insertResult; }; - const outcome = await this.#insertWithJsonParseRecovery( - flushId, - events, - doInsert, - `task_events_${this._version}` - ); + const outcome = await insertWithBadRowSkip({ + rows: events, + contextLabel, + logger, + logContext: { flushId, version: this._version }, + insert: (rows) => rawInsert(rows), + insertAllowingBadRows: (rows) => + rawInsert(rows, { + async_insert: 0, + input_format_parallel_parsing: 0, + input_format_allow_errors_num: String(rows.length), + input_format_allow_errors_ratio: 1, + }), + }); + this.#recordRecoveryOutcome(outcome, contextLabel, events.length); - if (outcome.kind === "dropped") { - // Loud log already emitted; nothing landed in ClickHouse — don't publish to Redis. + if (landedNothing(outcome, events.length)) { return; } logger.debug("ClickhouseEventRepository.flushBatch Inserted batch into clickhouse", { events: events.length, - insertResult: outcome.insertResult, - sanitized: outcome.kind === "sanitized", + outcome: outcome.kind, version: this._version, }); @@ -296,129 +346,60 @@ export class ClickhouseEventRepository implements IEventRepository { } async #flushLlmMetricsBatch(flushId: string, rows: LlmMetricsV1Input[]) { - const doInsert = async () => { - const [insertError, insertResult] = await this._clickhouse.llmMetrics.insert(rows, { + const rawInsert = async (batch: LlmMetricsV1Input[], extraSettings?: ClickHouseSettings) => { + const [insertError, insertResult] = await this._clickhouse.llmMetrics.insert(batch, { params: { - clickhouse_settings: this.#getClickhouseInsertSettings(), + clickhouse_settings: { ...this.#getClickhouseInsertSettings(), ...extraSettings }, }, }); if (insertError) throw insertError; return insertResult; }; - const outcome = await this.#insertWithJsonParseRecovery( - flushId, + const outcome = await insertWithBadRowSkip({ rows, - doInsert, - "llm_metrics_v1" - ); + contextLabel: "llm_metrics_v1", + logger, + logContext: { flushId }, + insert: (batch) => rawInsert(batch), + insertAllowingBadRows: (batch) => + rawInsert(batch, { + async_insert: 0, + input_format_parallel_parsing: 0, + input_format_allow_errors_num: String(batch.length), + input_format_allow_errors_ratio: 1, + }), + }); + this.#recordRecoveryOutcome(outcome, "llm_metrics_v1", rows.length); - if (outcome.kind === "dropped") { + if (landedNothing(outcome, rows.length)) { return; } logger.debug("ClickhouseEventRepository.flushLlmMetricsBatch Inserted LLM metrics batch", { rows: rows.length, - sanitized: outcome.kind === "sanitized", + outcome: outcome.kind, }); } - /** - * Wraps a ClickHouse insert callable with reactive UTF-16 sanitization. - * - * On a `Cannot parse JSON object` failure: - * 1. Sanitize the batch from `max(0, parsedRowN - 1)` onwards (rows - * before the failing one parsed fine — known good). - * 2. Retry the insert once with the sanitized batch. - * 3. If the retry still fails with the same error class, log loudly, - * increment `permanentlyDroppedBatches`, and return without - * throwing — the scheduler's transient-retry path would just repeat - * the same deterministic failure. - * - * Non-parse errors propagate unchanged so the scheduler's existing - * backoff/retry behaviour still handles transient network or CH issues. - */ - async #insertWithJsonParseRecovery<T extends object>( - flushId: string, - rows: T[], - doInsert: () => Promise<unknown>, - contextLabel: string - ): Promise< - | { kind: "inserted"; insertResult: unknown } - | { kind: "sanitized"; insertResult: unknown } - | { kind: "dropped" } - > { - try { - return { kind: "inserted", insertResult: await doInsert() }; - } catch (firstError) { - if (!isClickHouseJsonParseError(firstError)) throw firstError; - - const firstMessage = - typeof firstError === "object" && firstError !== null && "message" in firstError - ? String((firstError as { message?: unknown }).message ?? "") - : String(firstError); - - // Sanitize the whole batch. ClickHouse's `at row N` index is logged - // for observability but not used to slice — its semantics under - // parallel parsing are not stable enough to safely skip rows. - const rowHint = parseRowNumberFromError(firstMessage); - const { rowsTouched, fieldsSanitized } = sanitizeRows(rows); - - // Sanitizer found nothing to fix → retrying the exact same batch is - // guaranteed to hit the same deterministic parse failure. Skip the - // wasted ClickHouse round-trip and drop loudly. Throwing instead would - // hand the failure back to the scheduler's 3× transient-retry loop — - // exactly the retry storm this wrapper is designed to avoid. - if (fieldsSanitized === 0) { - this._permanentlyDroppedBatches += 1; - this._droppedBatchesCounter.add(1, { table: contextLabel }); - logger.error( - "Dropped batch — ClickHouse JSON parse error but sanitizer found nothing to fix", - { - flushId, - contextLabel, - batchSize: rows.length, - clickhouseRowHint: rowHint, - permanentlyDroppedBatches: this._permanentlyDroppedBatches, - sampleRow: JSON.stringify(rows[0] ?? null).slice(0, 1024), - clickhouseError: firstMessage.split("\n")[0], - } - ); - return { kind: "dropped" }; - } - - logger.warn("Sanitizing batch after ClickHouse JSON parse error", { - flushId, - contextLabel, - batchSize: rows.length, - clickhouseRowHint: rowHint, - rowsTouched, - fieldsSanitized, - clickhouseError: firstMessage.split("\n")[0], - }); + #recordRecoveryOutcome( + outcome: JsonParseRecoveryOutcome, + contextLabel: string, + batchSize: number + ) { + if (outcome.kind !== "recovered") { + return; + } - try { - return { kind: "sanitized", insertResult: await doInsert() }; - } catch (retryError) { - if (!isClickHouseJsonParseError(retryError)) throw retryError; + this._rowIsolationRecoveries += 1; + this._rowIsolatedBatchesCounter.add(1, { table: contextLabel }); + if (outcome.rowsDropped > 0) { + this._permanentlyDroppedRows += outcome.rowsDropped; + this._rowsDroppedCounter.add(outcome.rowsDropped, { table: contextLabel }); + if (outcome.rowsDroppedExact && outcome.rowsDropped === batchSize) { this._permanentlyDroppedBatches += 1; this._droppedBatchesCounter.add(1, { table: contextLabel }); - const retryMessage = - typeof retryError === "object" && retryError !== null && "message" in retryError - ? String((retryError as { message?: unknown }).message ?? "") - : String(retryError); - logger.error("Dropped batch after sanitize-retry still hit ClickHouse JSON parse error", { - flushId, - contextLabel, - batchSize: rows.length, - permanentlyDroppedBatches: this._permanentlyDroppedBatches, - sampleRow: JSON.stringify(rows[0] ?? null).slice(0, 1024), - firstError: firstMessage.split("\n")[0], - retryError: retryMessage.split("\n")[0], - }); - - return { kind: "dropped" }; } } } diff --git a/apps/webapp/app/v3/eventRepository/sanitizeRowsOnParseError.server.ts b/apps/webapp/app/v3/eventRepository/sanitizeRowsOnParseError.server.ts index 0374442200a..32d80ef43d4 100644 --- a/apps/webapp/app/v3/eventRepository/sanitizeRowsOnParseError.server.ts +++ b/apps/webapp/app/v3/eventRepository/sanitizeRowsOnParseError.server.ts @@ -81,6 +81,26 @@ export function parseRowNumberFromError(errorMessage: string): number | null { return match ? Number.parseInt(match[1], 10) : null; } +/** + * Extracts the failing-row index for the strip loop, where a wrong answer empties + * an innocent row's JSON instead of the poison one. + * + * ClickHouse reports the position as a parenthesised `(at row N)` suffix, but the + * same message embeds a snippet of the offending row's own data ahead of it + * (`Cannot parse JSON object here: {...}: (while reading ...): (at row N)`), so a + * task whose output merely contains the text `at row 7` would otherwise win the + * match. Requiring the parentheses and taking the LAST occurrence keeps the + * server's own suffix authoritative, since user data appears before it. + * + * Returns `null` when no parenthesised position is present; the caller treats + * that as "row not locatable" and bails to the skip insert rather than guessing. + */ +export function parseStrippableRowNumber(errorMessage: string): number | null { + const matches = [...errorMessage.matchAll(/\(at row (\d+)\)/g)]; + if (matches.length === 0) return null; + return Number.parseInt(matches[matches.length - 1][1], 10); +} + /** * Walks `value` recursively and replaces any string leaf that contains a * lone UTF-16 surrogate with `INVALID_UTF16_SENTINEL`. Mutates objects @@ -162,3 +182,454 @@ export function sanitizeRows<T extends object>(rows: T[]): SanitizeResult { return result; } + +export function errorMessage(err: unknown): string { + return typeof err === "object" && err !== null && "message" in err + ? String((err as { message?: unknown }).message ?? "") + : String(err); +} + +export function rawErrorMessage(err: unknown): string { + if (typeof err === "object" && err !== null) { + const raw = (err as { rawMessage?: unknown }).rawMessage; + if (typeof raw === "string" && raw.length > 0) return raw; + } + return errorMessage(err); +} + +/** + * `error` is part of the contract because losing a whole batch is a genuine + * failure an operator has to act on, and alerting watches error level. Partial + * recoveries, where the rest of the batch lands, stay at warn. + */ +export type JsonParseRecoveryLogger = { + info: (message: string, meta?: Record<string, unknown>) => void; + warn: (message: string, meta?: Record<string, unknown>) => void; + error: (message: string, meta?: Record<string, unknown>) => void; +}; + +/** + * Why recovery stopped isolating rows and fell back to a single `allow_errors` + * insert. Both causes land the same way, so they share the `capped` flag and its + * counter; this distinguishes them in logs. + */ +export type RecoveryBailReason = + /** The per-batch strip budget (`maxPoisonStrips`) was spent. A poison flood. */ + | "strip_budget_spent" + /** ClickHouse gave no usable `at row N` hint, so there was no row to strip. */ + | "row_not_locatable" + /** The loop guard tripped without either of the above. Should not happen. */ + | "strip_attempts_exhausted"; + +export type JsonParseRecoveryOutcome = + | { kind: "inserted"; insertResult: unknown } + | { kind: "sanitized"; insertResult: unknown } + | { + kind: "recovered"; + rowsStripped: number; + rowsDropped: number; + /** + * False when `rowsDropped` is a floor rather than an exact count, which is + * the case on tables with row-multiplying materialized views. Callers + * should read `rowsDropped` as "at least this many" when this is false. + */ + rowsDroppedExact: boolean; + capped: boolean; + bailReason?: RecoveryBailReason; + }; + +/** + * True when a recovery landed no rows at all, so nothing reached ClickHouse. + * + * Callers must not treat such a flush as a successful insert: there is no new + * data for subscribers to read, so notifying them only causes refetches that + * find nothing. Requires an exact dropped count, so a batch whose partial-drop + * count is only a floor is never mistaken for a total loss. + */ +export function landedNothing(outcome: JsonParseRecoveryOutcome, batchSize: number): boolean { + return ( + outcome.kind === "recovered" && outcome.rowsDroppedExact && outcome.rowsDropped >= batchSize + ); +} + +/** + * Default number of poison rows to isolate-and-strip precisely before bailing + * to a single `allow_errors` skip insert. One covers the common case (a single + * un-ingestable run in a flush) exactly, keeping that run's status. The bound + * matters because run-replication flushes are large (thousands of rows in prod) + * and stripping re-sends the whole batch once per poison row: without a small + * limit, a burst of un-ingestable runs in one flush would re-parse a large + * batch many times on the shared ClickHouse server. + */ +export const DEFAULT_MAX_POISON_STRIPS = 1; + +/** + * ClickHouse insert recovery for `Cannot parse JSON object` rejections on the + * runs table, where the poison run should KEEP its status (its row lands with + * its JSON column emptied) rather than be dropped. + * + * 1. Try the insert. Healthy batches pay zero recovery cost. + * 2. On a parse error, `sanitizeRows` losslessly repairs what it can in place + * (lone UTF-16 surrogates, out-of-range integers) and retries once. + * 3. If the sanitizer can't help, follow ClickHouse's `at row N` hint to the + * un-ingestable row and re-insert with the caller-selected JSON column(s) + * emptied via `stripJsonColumns`, up to `maxPoisonStrips` rows. Each + * stripped run still lands (keeps its terminal status); only the emptied + * content is lost. A row the caller's strip cannot make ingestable is + * reported again, makes no progress, and falls through to step 4. + * `insertSync` disables parallel parsing so `at row N` is reliable. + * 4. Cost bound: once `maxPoisonStrips` rows have been stripped and the batch + * STILL fails (or the failing row can't be located), stop stripping and + * land the batch with one `allow_errors` insert — the stripped rows and + * every clean row land in a single pass and the remaining un-ingestable + * rows are skipped. Recovery stays a fixed handful of inserts no matter how + * large or poisoned the batch is (`capped` marks that the bail was taken). + * 5. Non-parse errors propagate unchanged. + */ +export async function insertWithLimitedStrip<T extends object>(params: { + rows: T[]; + contextLabel: string; + logger: JsonParseRecoveryLogger; + logContext?: Record<string, unknown>; + insert: (rows: T[]) => Promise<unknown>; + insertSync: (rows: T[]) => Promise<unknown>; + insertAllowingBadRows: (rows: T[]) => Promise<unknown>; + stripJsonColumns: (row: T) => T; + maxPoisonStrips?: number; + hasMaterializedViews?: boolean; +}): Promise<JsonParseRecoveryOutcome> { + const { rows, contextLabel, logger, logContext, insert, insertSync, insertAllowingBadRows } = + params; + const stripJsonColumns = params.stripJsonColumns; + const maxPoisonStrips = params.maxPoisonStrips ?? DEFAULT_MAX_POISON_STRIPS; + const hasMaterializedViews = params.hasMaterializedViews ?? true; + + try { + return { kind: "inserted", insertResult: await insert(rows) }; + } catch (firstError) { + if (!isClickHouseJsonParseError(firstError)) throw firstError; + + const firstMessage = errorMessage(firstError); + const { rowsTouched, fieldsSanitized } = sanitizeRows(rows); + + if (fieldsSanitized > 0) { + logger.warn("Sanitizing batch after ClickHouse JSON parse error", { + ...logContext, + contextLabel, + batchSize: rows.length, + rowsTouched, + fieldsSanitized, + clickhouseError: firstMessage.split("\n")[0], + }); + + try { + return { kind: "sanitized", insertResult: await insert(rows) }; + } catch (retryError) { + if (!isClickHouseJsonParseError(retryError)) throw retryError; + } + } + + const working = rows.slice(); + const stripped = new Array(working.length).fill(false); + let rowsStripped = 0; + let bailReason: RecoveryBailReason = "strip_attempts_exhausted"; + + let guard = maxPoisonStrips + 2; + while (guard-- > 0) { + let parseError: unknown; + try { + await insertSync(working); + if (rowsStripped > 0) { + logger.info( + "Stripped un-ingestable rows after ClickHouse JSON parse error — batch landed with their JSON emptied", + { + ...logContext, + contextLabel, + batchSize: rows.length, + rowsStripped, + clickhouseError: firstMessage.split("\n")[0], + } + ); + } + return { + kind: "recovered", + rowsStripped, + rowsDropped: 0, + rowsDroppedExact: true, + capped: false, + }; + } catch (error) { + if (!isClickHouseJsonParseError(error)) throw error; + parseError = error; + } + + if (rowsStripped >= maxPoisonStrips) { + bailReason = "strip_budget_spent"; + break; + } + + const hint = parseStrippableRowNumber(rawErrorMessage(parseError)); + const index = hint === null ? -1 : hint - 1; + + if (index < 0 || index >= working.length || stripped[index]) { + bailReason = "row_not_locatable"; + break; + } + + working[index] = stripJsonColumns(working[index]); + stripped[index] = true; + rowsStripped += 1; + } + + const [skipError, insertResult] = await tryInsertAllowingBadRows( + insertAllowingBadRows, + working + ); + + if (skipError) { + return wholeBatchDropped({ + rows, + contextLabel, + logger, + logContext, + rowsStripped, + capped: true, + bailReason, + firstMessage, + skipError, + }); + } + + const dropped = droppedRowCount(insertResult, working.length, hasMaterializedViews); + + logger.warn("Landed the batch via allow_errors and skipped the remaining un-ingestable rows", { + ...logContext, + contextLabel, + bailReason, + batchSize: rows.length, + rowsStripped, + rowsDropped: dropped.rows, + rowsDroppedExact: dropped.exact, + landedRows: writtenRowCount(insertResult), + clickhouseError: firstMessage.split("\n")[0], + }); + + return { + kind: "recovered", + rowsStripped, + rowsDropped: dropped.rows, + rowsDroppedExact: dropped.exact, + capped: true, + bailReason, + }; + } +} + +/** + * Runs the `allow_errors` skip insert, separating the two ways it can fail. + * + * A surviving `Cannot parse JSON object` is deterministic: the same bytes will + * fail again, so it is returned for the caller to swallow and count. Anything + * else (a connection drop, a server restart) is transient and rethrown so the + * caller's retry layer still gets its chance. + */ +async function tryInsertAllowingBadRows<T extends object>( + insertAllowingBadRows: (rows: T[]) => Promise<unknown>, + rows: T[] +): Promise<[unknown, undefined] | [undefined, unknown]> { + try { + return [undefined, await insertAllowingBadRows(rows)]; + } catch (error) { + if (!isClickHouseJsonParseError(error)) throw error; + return [error, undefined]; + } +} + +/** + * Reports a batch that could not land even with `allow_errors`, at error level and + * without throwing. + * + * Handing a deterministic parse failure back to the caller's retry layer only + * burns the whole recovery again on bytes that cannot change, and in the event + * repository a terminal throw also skips the scheduler's queue-depth decrement, + * leaking a counter that feeds its load-shedding and memory-pressure decisions. + * Counting the batch as wholly dropped keeps both layers honest instead. + */ +function wholeBatchDropped<T extends object>(params: { + rows: T[]; + contextLabel: string; + logger: JsonParseRecoveryLogger; + logContext?: Record<string, unknown>; + rowsStripped: number; + capped: boolean; + bailReason?: RecoveryBailReason; + firstMessage: string; + skipError: unknown; +}): JsonParseRecoveryOutcome { + const { rows, contextLabel, logger, logContext, rowsStripped, capped, bailReason } = params; + + logger.error("Dropped the whole batch: ClickHouse rejected it even with allow_errors", { + ...logContext, + contextLabel, + bailReason, + batchSize: rows.length, + rowsStripped, + rowsDropped: rows.length, + clickhouseError: params.firstMessage.split("\n")[0], + skipInsertError: errorMessage(params.skipError).split("\n")[0], + }); + + return { + kind: "recovered", + rowsStripped, + rowsDropped: rows.length, + rowsDroppedExact: true, + capped, + bailReason, + }; +} + +function writtenRowCount(insertResult: unknown): number | null { + if (typeof insertResult === "object" && insertResult !== null) { + const summary = (insertResult as { summary?: { written_rows?: unknown } }).summary; + const written = summary?.written_rows; + if (typeof written === "number" && Number.isFinite(written)) return written; + if (typeof written === "string" && written.length > 0) { + const parsed = Number.parseInt(written, 10); + if (Number.isFinite(parsed)) return parsed; + } + } + return null; +} + +/** + * Reaching an `allow_errors` insert means the batch still failed to parse after + * everything cheaper was tried, so at least one row is un-ingestable and will be + * skipped. This is the floor we report when the exact count isn't derivable — + * reporting 0 there would read as "no data lost", which is the opposite of what + * happened. + */ +const MINIMUM_DROPPED_ROWS = 1; + +/** + * Derives how many rows an `allow_errors` insert dropped, from the insert + * summary's `written_rows`. + * + * Exact only when `written_rows` is a clean count of the target-table rows. On + * tables with row-multiplying materialized views ClickHouse folds the MV-written + * rows into `written_rows` too, so a partial-drop count isn't derivable; the only + * exact signal there is `written_rows === 0`, meaning the whole batch was dropped + * (no base rows, so no MV rows either). Everywhere else on such a table the + * result is the `MINIMUM_DROPPED_ROWS` floor flagged `exact: false`, so callers + * can say "at least one" instead of a confident zero. + */ +function droppedRowCount( + insertResult: unknown, + batchSize: number, + hasMaterializedViews: boolean +): { rows: number; exact: boolean } { + const written = writtenRowCount(insertResult); + if (written === null) return { rows: MINIMUM_DROPPED_ROWS, exact: false }; + if (written === 0) return { rows: batchSize, exact: true }; + if (hasMaterializedViews) return { rows: MINIMUM_DROPPED_ROWS, exact: false }; + return { rows: Math.max(0, batchSize - written), exact: true }; +} + +/** + * Shared ClickHouse insert recovery that SKIPS un-ingestable rows, for the + * high-volume append-only tables (trace events, run payloads) where dropping a + * single un-ingestable row is acceptable and precise row isolation isn't worth + * its re-parse cost on the shared ClickHouse server. + * + * 1. Try the insert. Healthy batches pay zero recovery cost. + * 2. On a parse error, `sanitizeRows` losslessly repairs what it can in place + * and retries once, so a repairable row still lands in full. + * 3. If the sanitizer can't help, re-insert once with ClickHouse's + * `input_format_allow_errors_*` so the good rows land in a single pass and + * only the un-ingestable rows are skipped. A poison flood costs one extra + * insert regardless of how many rows are bad. + * 4. Non-parse errors propagate unchanged. + * + * The batch-level recovery is always counted by the caller. The per-row dropped + * count is exact on tables without row-multiplying materialized views; on tables + * that have them it is a floor of one flagged `rowsDroppedExact: false`, since + * ClickHouse's summary can't separate skipped base rows from MV rows (see + * `droppedRowCount`). Reaching the skip insert always means real data loss, so it + * logs at warn. + */ +export async function insertWithBadRowSkip<T extends object>(params: { + rows: T[]; + contextLabel: string; + logger: JsonParseRecoveryLogger; + logContext?: Record<string, unknown>; + insert: (rows: T[]) => Promise<unknown>; + insertAllowingBadRows: (rows: T[]) => Promise<unknown>; + hasMaterializedViews?: boolean; +}): Promise<JsonParseRecoveryOutcome> { + const { rows, contextLabel, logger, logContext, insert, insertAllowingBadRows } = params; + const hasMaterializedViews = params.hasMaterializedViews ?? true; + + try { + return { kind: "inserted", insertResult: await insert(rows) }; + } catch (firstError) { + if (!isClickHouseJsonParseError(firstError)) throw firstError; + + const firstMessage = errorMessage(firstError); + const { rowsTouched, fieldsSanitized } = sanitizeRows(rows); + + if (fieldsSanitized > 0) { + logger.warn("Sanitizing batch after ClickHouse JSON parse error", { + ...logContext, + contextLabel, + batchSize: rows.length, + rowsTouched, + fieldsSanitized, + clickhouseError: firstMessage.split("\n")[0], + }); + + try { + return { kind: "sanitized", insertResult: await insert(rows) }; + } catch (retryError) { + if (!isClickHouseJsonParseError(retryError)) throw retryError; + } + } + + const [skipError, insertResult] = await tryInsertAllowingBadRows(insertAllowingBadRows, rows); + + if (skipError) { + return wholeBatchDropped({ + rows, + contextLabel, + logger, + logContext, + rowsStripped: 0, + capped: false, + firstMessage, + skipError, + }); + } + + const dropped = droppedRowCount(insertResult, rows.length, hasMaterializedViews); + + logger.warn( + "Skipped un-ingestable rows after ClickHouse JSON parse error — landed the rest of the batch", + { + ...logContext, + contextLabel, + batchSize: rows.length, + rowsDropped: dropped.rows, + rowsDroppedExact: dropped.exact, + landedRows: writtenRowCount(insertResult), + clickhouseError: firstMessage.split("\n")[0], + } + ); + + return { + kind: "recovered", + rowsStripped: 0, + rowsDropped: dropped.rows, + rowsDroppedExact: dropped.exact, + capped: false, + }; + } +} diff --git a/apps/webapp/test/clickhouseEventRepositoryJsonRecovery.test.ts b/apps/webapp/test/clickhouseEventRepositoryJsonRecovery.test.ts new file mode 100644 index 00000000000..375d7c4498d --- /dev/null +++ b/apps/webapp/test/clickhouseEventRepositoryJsonRecovery.test.ts @@ -0,0 +1,125 @@ +import { ClickHouse, type TaskEventV2Input } from "@internal/clickhouse"; +import { clickhouseTest } from "@internal/testcontainers"; +import { describe, expect } from "vitest"; +import { z } from "zod"; +import { + ClickhouseEventRepository, + convertDateToClickhouseDateTime, +} from "~/v3/eventRepository/clickhouseEventRepository.server"; + +const TIMEOUT_MS = 60_000; + +function deeplyNested(depth: number): Record<string, unknown> { + let node: Record<string, unknown> = { leaf: 1 }; + for (let i = 0; i < depth; i++) { + node = { [`k${i}`]: node }; + } + return node; +} + +function startTime(baseMs: number, offsetMs: number): string { + const ns = ((BigInt(baseMs) + BigInt(offsetMs)) * 1_000_000n).toString(); + return `${ns.substring(0, 10)}.${ns.substring(10)}`; +} + +describe("ClickhouseEventRepository JSON parse recovery", () => { + clickhouseTest( + "lands the good events and skips the poison event when one event has ClickHouse-unparseable attributes", + async ({ clickhouseContainer }) => { + const clickhouse = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + logLevel: "error", + }); + + const repository = new ClickhouseEventRepository({ + clickhouse, + version: "v2", + insertStrategy: "insert", + batchSize: 100, + flushInterval: 200, + }); + + const environmentId = "env_json_recovery_test"; + const organizationId = "org_json_recovery_test"; + const projectId = "proj_json_recovery_test"; + const traceId = "b".repeat(32); + const baseMs = Date.now(); + const expiresAt = convertDateToClickhouseDateTime( + new Date(baseMs + 365 * 24 * 60 * 60 * 1000) + ); + + function makeRow(i: number, attributes: unknown): TaskEventV2Input { + return { + environment_id: environmentId, + organization_id: organizationId, + project_id: projectId, + task_identifier: "json-recovery-task", + run_id: `run_${i}`, + start_time: startTime(baseMs, i), + duration: "1000000", + trace_id: traceId, + span_id: `span_recovery_${String(i).padStart(6, "0")}`, + parent_span_id: "", + message: `event ${i}`, + kind: "SPAN", + status: "OK", + attributes, + metadata: "{}", + expires_at: expiresAt, + }; + } + + const goodSpanIds: string[] = []; + let poisonSpanId = ""; + const rows: TaskEventV2Input[] = []; + for (let i = 0; i < 5; i++) { + const isPoison = i === 2; + const row = makeRow(i, isPoison ? deeplyNested(1500) : { ok: true, i }); + rows.push(row); + if (isPoison) { + poisonSpanId = row.span_id; + } else { + goodSpanIds.push(row.span_id); + } + } + + try { + (repository as any).addToBatch(rows); + + const queryEvents = clickhouse.reader.query({ + name: "event-recovery-check", + query: + "SELECT span_id, toJSONString(attributes) AS attributes_json FROM trigger_dev.task_events_v2 WHERE environment_id = {env_id:String}", + schema: z.object({ span_id: z.string(), attributes_json: z.string() }), + params: z.object({ env_id: z.string() }), + }); + + const rowsById = await vi.waitFor( + async () => { + const [queryError, resultRows] = await queryEvents({ env_id: environmentId }); + expect(queryError).toBeNull(); + const byId = new Map((resultRows ?? []).map((r) => [r.span_id, r])); + for (const id of goodSpanIds) { + expect(byId.has(id)).toBe(true); + } + return byId; + }, + { timeout: 30_000, interval: 250 } + ); + + for (const id of goodSpanIds) { + expect(rowsById.get(id)!.attributes_json).toContain('"ok":true'); + } + + expect(rowsById.has(poisonSpanId)).toBe(false); + + expect(repository.permanentlyDroppedBatches).toBe(0); + expect(repository.rowIsolationRecoveries).toBeGreaterThanOrEqual(1); + expect(repository.permanentlyDroppedRows).toBeGreaterThanOrEqual(1); + } finally { + await (repository as any)._flushScheduler?.shutdown?.(); + } + }, + TIMEOUT_MS + ); +}); diff --git a/apps/webapp/test/runsReplicationBenchmark.producer.ts b/apps/webapp/test/runsReplicationBenchmark.producer.ts index 547d1318c46..3f34d5c9f80 100644 --- a/apps/webapp/test/runsReplicationBenchmark.producer.ts +++ b/apps/webapp/test/runsReplicationBenchmark.producer.ts @@ -15,6 +15,17 @@ interface ProducerConfig { numRuns: number; errorRate: number; // 0.07 = 7% batchSize: number; + poisonRate?: number; + poisonDepth?: number; +} + +function deeplyNestedJson(depth: number): string { + const safeDepth = Number.isSafeInteger(depth) && depth >= 0 ? depth : 0; + const open: string[] = []; + for (let i = 0; i < safeDepth; i++) { + open.push(`{"k${i}":`); + } + return open.join("") + '{"leaf":1}' + "}".repeat(safeDepth); } // Error templates for realistic variety @@ -108,6 +119,9 @@ async function runProducer(config: ProducerConfig) { const startTime = performance.now(); let created = 0; let withErrors = 0; + let poisoned = 0; + const poisonRate = config.poisonRate ?? 0; + const poisonDepth = config.poisonDepth ?? 1500; // Process in batches to avoid overwhelming the database for (let batch = 0; batch < Math.ceil(config.numRuns / config.batchSize); batch++) { @@ -120,6 +134,8 @@ async function runProducer(config: ProducerConfig) { const hasError = Math.random() < config.errorRate; const status = hasError ? "COMPLETED_WITH_ERRORS" : "COMPLETED_SUCCESSFULLY"; + const isPoison = Math.random() < poisonRate; + const runData: any = { friendlyId: `run_bench_${Date.now()}_${i}`, taskIdentifier: `benchmark-task-${i % 10}`, // Vary task identifiers @@ -142,6 +158,12 @@ async function runProducer(config: ProducerConfig) { withErrors++; } + if (isPoison) { + runData.output = deeplyNestedJson(poisonDepth); + runData.outputType = "application/json"; + poisoned++; + } + runs.push(runData); } @@ -168,6 +190,7 @@ async function runProducer(config: ProducerConfig) { console.log(`[Producer] Completed:`); console.log(` - Total runs: ${created}`); console.log(` - With errors: ${withErrors} (${((withErrors / created) * 100).toFixed(1)}%)`); + console.log(` - Poisoned: ${poisoned} (${((poisoned / created) * 100).toFixed(1)}%)`); console.log(` - Duration: ${duration.toFixed(0)}ms`); console.log(` - Throughput: ${throughput.toFixed(0)} runs/sec`); @@ -178,6 +201,7 @@ async function runProducer(config: ProducerConfig) { stats: { created, withErrors, + poisoned, duration, throughput, }, diff --git a/apps/webapp/test/runsReplicationJsonRecoveryBenchmark.test.ts b/apps/webapp/test/runsReplicationJsonRecoveryBenchmark.test.ts new file mode 100644 index 00000000000..7e97dba5d14 --- /dev/null +++ b/apps/webapp/test/runsReplicationJsonRecoveryBenchmark.test.ts @@ -0,0 +1,287 @@ +import { ClickHouse } from "@internal/clickhouse"; +import { replicationContainerTest } from "@internal/testcontainers"; +import { fork } from "node:child_process"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { setTimeout } from "node:timers/promises"; +import { z } from "zod"; +import { RunsReplicationService } from "~/services/runsReplicationService.server"; +import { TestReplicationClickhouseFactory } from "./utils/testReplicationClickhouseFactory"; +import { createInMemoryMetrics, createInMemoryTracing } from "./utils/tracing"; + +vi.setConfig({ testTimeout: 300_000 }); + +const CONFIG = { + NUM_RUNS: parseInt(process.env.BENCHMARK_NUM_RUNS || "5000", 10), + PRODUCER_BATCH_SIZE: 100, + FLUSH_BATCH_SIZE: 50, + FLUSH_INTERVAL_MS: 100, + MAX_FLUSH_CONCURRENCY: 4, + REPLICATION_TIMEOUT_MS: 180_000, + POISON_RATE: parseFloat(process.env.BENCHMARK_POISON_RATE || "0.05"), +}; + +class ELUMonitor { + private samples: number[] = []; + private interval: NodeJS.Timeout | null = null; + + start(intervalMs = 100) { + this.samples = []; + let last = performance.eventLoopUtilization(); + this.interval = setInterval(() => { + const current = performance.eventLoopUtilization(); + this.samples.push(performance.eventLoopUtilization(current, last).utilization * 100); + last = current; + }, intervalMs); + } + + stop() { + if (this.interval) { + clearInterval(this.interval); + this.interval = null; + } + if (this.samples.length === 0) return { mean: 0, p50: 0, p95: 0, p99: 0, samples: 0 }; + const sorted = [...this.samples].sort((a, b) => a - b); + const mean = sorted.reduce((s, v) => s + v, 0) / sorted.length; + return { + mean, + p50: sorted[Math.floor(sorted.length * 0.5)], + p95: sorted[Math.floor(sorted.length * 0.95)], + p99: sorted[Math.floor(sorted.length * 0.99)], + samples: sorted.length, + }; + } +} + +function runProducer(config: { + postgresUrl: string; + organizationId: string; + projectId: string; + environmentId: string; + numRuns: number; + errorRate: number; + batchSize: number; + poisonRate: number; + poisonDepth: number; +}): Promise<{ created: number; withErrors: number; poisoned: number; duration: number }> { + return new Promise((resolve, reject) => { + const producerPath = path.join(__dirname, "runsReplicationBenchmark.producer.ts"); + const child = fork(producerPath, [JSON.stringify(config)], { + stdio: ["ignore", "pipe", "pipe", "ipc"], + execArgv: ["-r", "tsx/cjs"], + }); + child.stdout?.on("data", (d) => console.log(d.toString().trim())); + child.stderr?.on("data", (d) => console.error(d.toString().trim())); + child.on("message", (m: any) => { + if (m.type === "complete") resolve(m.stats); + else if (m.type === "error") reject(new Error(m.error)); + }); + child.on("error", reject); + child.on("exit", (code) => { + if (code !== 0) reject(new Error(`Producer exited with code ${code}`)); + }); + }); +} + +async function waitForCount( + clickhouse: ClickHouse, + organizationId: string, + expectedCount: number, + timeoutMs: number +): Promise<{ duration: number; count: number }> { + const startTime = performance.now(); + const deadline = startTime + timeoutMs; + const queryRuns = clickhouse.reader.query({ + name: "bench-count", + query: + "SELECT count(*) as count FROM trigger_dev.task_runs_v2 WHERE organization_id = {org_id:String}", + schema: z.object({ count: z.number() }), + params: z.object({ org_id: z.string() }), + }); + while (performance.now() < deadline) { + const [error, result] = await queryRuns({ org_id: organizationId }); + if (error) throw new Error(`Failed to query ClickHouse: ${error.message}`); + const count = result?.[0]?.count || 0; + if (count >= expectedCount) return { duration: performance.now() - startTime, count }; + await setTimeout(500); + } + const [, result] = await queryRuns({ org_id: organizationId }); + return { duration: performance.now() - startTime, count: result?.[0]?.count || 0 }; +} + +async function runScenario( + name: string, + poisonRate: number, + ctx: { clickhouseContainer: any; redisOptions: any; postgresContainer: any; prisma: any } +) { + const { clickhouseContainer, redisOptions, postgresContainer, prisma } = ctx; + + const organization = await prisma.organization.create({ + data: { title: `bench-${name}`, slug: `bench-${name}` }, + }); + const project = await prisma.project.create({ + data: { + name: `bench-${name}`, + slug: `bench-${name}`, + organizationId: organization.id, + externalRef: `bench-${name}`, + }, + }); + const runtimeEnvironment = await prisma.runtimeEnvironment.create({ + data: { + slug: `bench-${name}`, + type: "DEVELOPMENT", + projectId: project.id, + organizationId: organization.id, + apiKey: `bench-${name}`, + pkApiKey: `bench-${name}`, + shortcode: `bench-${name}`, + }, + }); + + const clickhouse = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: `bench-${name}`, + compression: { request: true }, + logLevel: "error", + }); + + const { tracer } = createInMemoryTracing(); + const metricsHelper = createInMemoryMetrics(); + + const service = new RunsReplicationService({ + clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse), + pgConnectionUrl: postgresContainer.getConnectionUri(), + serviceName: `bench-${name}`, + slotName: `bench_${name.replace(/-/g, "_")}`, + publicationName: `bench_${name.replace(/-/g, "_")}_pub`, + redisOptions, + maxFlushConcurrency: CONFIG.MAX_FLUSH_CONCURRENCY, + flushIntervalMs: CONFIG.FLUSH_INTERVAL_MS, + flushBatchSize: CONFIG.FLUSH_BATCH_SIZE, + maxPoisonStripsPerBatch: CONFIG.NUM_RUNS, + leaderLockTimeoutMs: 10000, + leaderLockExtendIntervalMs: 2000, + ackIntervalSeconds: 10, + tracer, + meter: metricsHelper.meter, + logLevel: "error", + }); + + await service.start(); + + const elu = new ELUMonitor(); + elu.start(100); + + let producerStats!: { created: number; withErrors: number; poisoned: number; duration: number }; + let replication!: { duration: number; count: number }; + let eluStats!: ReturnType<ELUMonitor["stop"]>; + + try { + producerStats = await runProducer({ + postgresUrl: postgresContainer.getConnectionUri(), + organizationId: organization.id, + projectId: project.id, + environmentId: runtimeEnvironment.id, + numRuns: CONFIG.NUM_RUNS, + errorRate: 0.07, + batchSize: CONFIG.PRODUCER_BATCH_SIZE, + poisonRate, + poisonDepth: 1500, + }); + + const expectedLanded = producerStats.created; + replication = await waitForCount( + clickhouse, + organization.id, + expectedLanded, + CONFIG.REPLICATION_TIMEOUT_MS + ); + } finally { + eluStats = elu.stop(); + await service.stop(); + await metricsHelper.shutdown(); + } + + const throughput = (replication.count / replication.duration) * 1000; + const expectedLanded = producerStats.created; + + console.log(`\n${"=".repeat(72)}`); + console.log(`SCENARIO: ${name} (poison rate ${(poisonRate * 100).toFixed(1)}%)`); + console.log(`${"=".repeat(72)}`); + console.log(`Produced: ${producerStats.created} runs (${producerStats.poisoned} poisoned)`); + console.log(`Landed in CH: ${replication.count} / expected ${expectedLanded}`); + console.log(`Repl dur: ${replication.duration.toFixed(0)}ms`); + console.log(`Throughput: ${throughput.toFixed(0)} runs/sec`); + console.log(`Row-isolation recoveries: ${service.rowIsolationRecoveries}`); + console.log(`Recovery cap hits: ${service.recoveryCapHits}`); + console.log(`Rows stripped (kept row, lost JSON): ${service.rowsStripped}`); + console.log(`Rows dropped (unrecoverable): ${service.permanentlyDroppedRows}`); + console.log(`Dropped batches (whole): ${service.permanentlyDroppedBatches}`); + console.log( + `ELU mean=${eluStats.mean.toFixed(2)}% p50=${eluStats.p50.toFixed(2)}% p95=${eluStats.p95.toFixed(2)}% p99=${eluStats.p99.toFixed(2)}% (${eluStats.samples} samples)` + ); + console.log(`${"=".repeat(72)}\n`); + + return { + name, + poisonRate, + producerStats, + replication, + eluStats, + throughput, + expectedLanded, + service, + }; +} + +describe("RunsReplicationService JSON-recovery ELU benchmark", () => { + replicationContainerTest.skipIf(process.env.BENCHMARKS_ENABLED !== "1")( + "measures ELU on the healthy hot path vs the row-isolation recovery path", + async ({ clickhouseContainer, redisOptions, postgresContainer, prisma }) => { + await prisma.$executeRawUnsafe(`ALTER TABLE public."TaskRun" REPLICA IDENTITY FULL;`); + + const healthy = await runScenario("healthy-0pct", 0, { + clickhouseContainer, + redisOptions, + postgresContainer, + prisma, + }); + + const poisoned = await runScenario("poison", CONFIG.POISON_RATE, { + clickhouseContainer, + redisOptions, + postgresContainer, + prisma, + }); + + const eluMeanDelta = + ((poisoned.eluStats.mean - healthy.eluStats.mean) / healthy.eluStats.mean) * 100; + const eluP99Delta = + ((poisoned.eluStats.p99 - healthy.eluStats.p99) / healthy.eluStats.p99) * 100; + + console.log(`\n${"=".repeat(72)}`); + console.log("COMPARISON (healthy 0% -> poison)"); + console.log(`${"=".repeat(72)}`); + console.log( + `ELU mean: ${healthy.eluStats.mean.toFixed(2)}% -> ${poisoned.eluStats.mean.toFixed(2)}% (${eluMeanDelta > 0 ? "+" : ""}${eluMeanDelta.toFixed(1)}%)` + ); + console.log( + `ELU p99: ${healthy.eluStats.p99.toFixed(2)}% -> ${poisoned.eluStats.p99.toFixed(2)}% (${eluP99Delta > 0 ? "+" : ""}${eluP99Delta.toFixed(1)}%)` + ); + console.log(`${"=".repeat(72)}\n`); + + expect(healthy.replication.count).toBe(healthy.expectedLanded); + expect(healthy.service.rowIsolationRecoveries).toBe(0); + expect(healthy.service.rowsStripped).toBe(0); + expect(healthy.service.permanentlyDroppedBatches).toBe(0); + + expect(poisoned.replication.count).toBe(poisoned.expectedLanded); + expect(poisoned.service.permanentlyDroppedBatches).toBe(0); + expect(poisoned.service.permanentlyDroppedRows).toBe(0); + expect(poisoned.service.recoveryCapHits).toBe(0); + expect(poisoned.service.rowIsolationRecoveries).toBeGreaterThanOrEqual(1); + expect(poisoned.service.rowsStripped).toBe(poisoned.producerStats.poisoned); + } + ); +}); diff --git a/apps/webapp/test/runsReplicationService.part10.test.ts b/apps/webapp/test/runsReplicationService.part10.test.ts new file mode 100644 index 00000000000..cab9cef8657 --- /dev/null +++ b/apps/webapp/test/runsReplicationService.part10.test.ts @@ -0,0 +1,230 @@ +import { ClickHouse } from "@internal/clickhouse"; +import { replicationContainerTest } from "@internal/testcontainers"; +import { z } from "zod"; +import { RunsReplicationService } from "~/services/runsReplicationService.server"; +import { TestReplicationClickhouseFactory } from "./utils/testReplicationClickhouseFactory"; +import { createInMemoryTracing } from "./utils/tracing"; + +vi.setConfig({ testTimeout: 60_000 }); + +function deeplyNested(depth: number): Record<string, unknown> { + let node: Record<string, unknown> = { leaf: 1 }; + for (let i = 0; i < depth; i++) { + node = { [`k${i}`]: node }; + } + return node; +} + +function createService( + clickhouse: ClickHouse, + postgresUrl: string, + redisOptions: any, + flushIntervalMs = 500 +) { + const { tracer } = createInMemoryTracing(); + return new RunsReplicationService({ + clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse), + pgConnectionUrl: postgresUrl, + serviceName: "runs-replication", + slotName: "task_runs_to_clickhouse_v1", + publicationName: "task_runs_to_clickhouse_v1_publication", + redisOptions, + maxFlushConcurrency: 1, + flushIntervalMs, + flushBatchSize: 50, + leaderLockTimeoutMs: 5000, + leaderLockExtendIntervalMs: 1000, + ackIntervalSeconds: 5, + tracer, + logLevel: "warn", + }); +} + +async function setupProject(prisma: any) { + const organization = await prisma.organization.create({ data: { title: "test", slug: "test" } }); + const project = await prisma.project.create({ + data: { name: "test", slug: "test", organizationId: organization.id, externalRef: "test" }, + }); + const runtimeEnvironment = await prisma.runtimeEnvironment.create({ + data: { + slug: "test", + type: "DEVELOPMENT", + projectId: project.id, + organizationId: organization.id, + apiKey: "test", + pkApiKey: "test", + shortcode: "test", + }, + }); + return { organization, project, runtimeEnvironment }; +} + +async function createRun( + prisma: any, + ctx: { organization: any; project: any; runtimeEnvironment: any }, + i: number, + isPoison: boolean +) { + return prisma.taskRun.create({ + data: { + friendlyId: `run_batchdrop_${i}`, + taskIdentifier: "my-task", + payload: JSON.stringify({ i }), + payloadType: "application/json", + output: isPoison ? JSON.stringify(deeplyNested(1500)) : JSON.stringify({ ok: true, i }), + outputType: "application/json", + traceId: `trace_${i}`, + spanId: `span_${i}`, + queue: "test", + status: "COMPLETED_SUCCESSFULLY", + runtimeEnvironmentId: ctx.runtimeEnvironment.id, + projectId: ctx.project.id, + organizationId: ctx.organization.id, + environmentType: "DEVELOPMENT", + engine: "V2", + }, + }); +} + +describe("RunsReplicationService (part 10/10) — JSON parse recovery", () => { + replicationContainerTest( + "strips a single poison run and lands every run (poison run keeps its status, output stripped)", + async ({ clickhouseContainer, redisOptions, postgresContainer, prisma }) => { + await prisma.$executeRawUnsafe(`ALTER TABLE public."TaskRun" REPLICA IDENTITY FULL;`); + + const clickhouse = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "runs-replication", + compression: { request: true }, + logLevel: "warn", + }); + + const runsReplicationService = createService( + clickhouse, + postgresContainer.getConnectionUri(), + redisOptions + ); + await runsReplicationService.start(); + + const ctx = await setupProject(prisma); + + const goodRunIds: string[] = []; + let poisonRunId = ""; + for (let i = 0; i < 5; i++) { + const isPoison = i === 2; + const run = await createRun(prisma, ctx, i, isPoison); + if (isPoison) poisonRunId = run.id; + else goodRunIds.push(run.id); + } + + const queryRuns = clickhouse.reader.query({ + name: "runs-replication-batchdrop", + query: + "SELECT run_id, status, toJSONString(output) AS output_json FROM trigger_dev.task_runs_v2 FINAL WHERE organization_id = {org_id:String}", + schema: z.object({ run_id: z.string(), status: z.string(), output_json: z.string() }), + params: z.object({ org_id: z.string() }), + }); + + const rowsById = await vi.waitFor( + async () => { + const [queryError, rows] = await queryRuns({ org_id: ctx.organization.id }); + expect(queryError).toBeNull(); + const byId = new Map((rows ?? []).map((r) => [r.run_id, r])); + for (const id of [...goodRunIds, poisonRunId]) { + expect(byId.has(id)).toBe(true); + } + return byId; + }, + { timeout: 30_000, interval: 250 } + ); + + for (const id of goodRunIds) { + expect(rowsById.get(id)!.output_json).toContain('"ok":true'); + } + + const poison = rowsById.get(poisonRunId)!; + expect(poison.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(poison.output_json).toBe("{}"); + + expect(runsReplicationService.permanentlyDroppedBatches).toBe(0); + expect(runsReplicationService.permanentlyDroppedRows).toBe(0); + expect(runsReplicationService.recoveryCapHits).toBe(0); + expect(runsReplicationService.rowIsolationRecoveries).toBeGreaterThanOrEqual(1); + expect(runsReplicationService.rowsStripped).toBeGreaterThanOrEqual(1); + + await runsReplicationService.stop(); + } + ); + + replicationContainerTest( + "strips up to the limit then skips the excess poison via allow_errors (2 poison, limit 1)", + async ({ clickhouseContainer, redisOptions, postgresContainer, prisma }) => { + await prisma.$executeRawUnsafe(`ALTER TABLE public."TaskRun" REPLICA IDENTITY FULL;`); + + const clickhouse = new ClickHouse({ + url: clickhouseContainer.getConnectionUrl(), + name: "runs-replication", + compression: { request: true }, + logLevel: "warn", + }); + + const runsReplicationService = createService( + clickhouse, + postgresContainer.getConnectionUri(), + redisOptions, + 3000 + ); + await runsReplicationService.start(); + + const ctx = await setupProject(prisma); + + const goodRunIds: string[] = []; + const poisonRunIds: string[] = []; + for (let i = 0; i < 5; i++) { + const isPoison = i === 1 || i === 3; + const run = await createRun(prisma, ctx, i, isPoison); + if (isPoison) poisonRunIds.push(run.id); + else goodRunIds.push(run.id); + } + + const queryRuns = clickhouse.reader.query({ + name: "runs-replication-bail", + query: + "SELECT run_id, status, toJSONString(output) AS output_json FROM trigger_dev.task_runs_v2 FINAL WHERE organization_id = {org_id:String}", + schema: z.object({ run_id: z.string(), status: z.string(), output_json: z.string() }), + params: z.object({ org_id: z.string() }), + }); + + const rowsById = await vi.waitFor( + async () => { + const [queryError, rows] = await queryRuns({ org_id: ctx.organization.id }); + expect(queryError).toBeNull(); + const byId = new Map((rows ?? []).map((r) => [r.run_id, r])); + for (const id of goodRunIds) { + expect(byId.has(id)).toBe(true); + } + expect(runsReplicationService.recoveryCapHits).toBeGreaterThanOrEqual(1); + return byId; + }, + { timeout: 30_000, interval: 250 } + ); + + for (const id of goodRunIds) { + expect(rowsById.get(id)!.output_json).toContain('"ok":true'); + } + + const landedPoison = poisonRunIds.filter((id) => rowsById.has(id)); + expect(landedPoison).toHaveLength(1); + const strippedPoison = rowsById.get(landedPoison[0]!)!; + expect(strippedPoison.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(strippedPoison.output_json).toBe("{}"); + + expect(runsReplicationService.permanentlyDroppedBatches).toBe(0); + expect(runsReplicationService.rowsStripped).toBeGreaterThanOrEqual(1); + expect(runsReplicationService.recoveryCapHits).toBeGreaterThanOrEqual(1); + expect(runsReplicationService.permanentlyDroppedRows).toBeGreaterThanOrEqual(1); + + await runsReplicationService.stop(); + } + ); +}); diff --git a/apps/webapp/test/sanitizeRowsOnParseError.test.ts b/apps/webapp/test/sanitizeRowsOnParseError.test.ts index e3353176624..60092e3a29c 100644 --- a/apps/webapp/test/sanitizeRowsOnParseError.test.ts +++ b/apps/webapp/test/sanitizeRowsOnParseError.test.ts @@ -1,8 +1,12 @@ import { describe, it, expect } from "vitest"; import { INVALID_UTF16_SENTINEL, + insertWithBadRowSkip, + insertWithLimitedStrip, isClickHouseJsonParseError, + landedNothing, parseRowNumberFromError, + parseStrippableRowNumber, sanitizeRows, sanitizeUnknownInPlace, } from "~/v3/eventRepository/sanitizeRowsOnParseError.server"; @@ -10,6 +14,58 @@ import { const HIGH_SURROGATE = "\uD800"; const LOW_SURROGATE = "\uDC00"; +type FakeRow = { + id: number; + poison?: boolean; + unstrippable?: boolean; + stripped?: boolean; + msg?: string; +}; + +const silentLogger = { info: () => {}, warn: () => {}, error: () => {} }; + +function parseErrorAtRow(oneBasedRow: number) { + return new Error( + `Cannot parse JSON object here: {...}: (at row ${oneBasedRow})\n: While executing ParallelParsingBlockInputFormat.` + ); +} + +/** + * Builds insert doubles for `insertWithLimitedStrip`: an insert that fails at + * the first still-poison row (reporting ClickHouse's 1-based `at row N`), an + * allow-bad-rows insert that lands only the good rows and reports the landed + * count in a summary, and a strip that clears the poison marker (but cannot fix + * an `unstrippable` row). + */ +function makeHarness() { + const landed: FakeRow[] = []; + let insertCalls = 0; + let allowCalls = 0; + const insertSync = async (rows: FakeRow[]) => { + insertCalls += 1; + const badIndex = rows.findIndex((r) => r.poison || r.unstrippable); + if (badIndex >= 0) throw parseErrorAtRow(badIndex + 1); + landed.push(...rows); + return { summary: { written_rows: String(rows.length) } }; + }; + const insertAllowingBadRows = async (rows: FakeRow[]) => { + allowCalls += 1; + const good = rows.filter((r) => !(r.poison || r.unstrippable)); + landed.push(...good); + return { summary: { written_rows: String(good.length) } }; + }; + const stripJsonColumns = (row: FakeRow): FakeRow => + row.unstrippable ? row : { ...row, poison: false, stripped: true }; + return { + landed, + insertSync, + insertAllowingBadRows, + stripJsonColumns, + insertCalls: () => insertCalls, + allowCalls: () => allowCalls, + }; +} + describe("isClickHouseJsonParseError", () => { it("recognises ClickHouse's parse-error string", () => { const err = new Error( @@ -52,6 +108,86 @@ describe("parseRowNumberFromError", () => { }); }); +describe("landedNothing", () => { + it("is true only when an exact dropped count covers the whole batch", () => { + expect( + landedNothing( + { + kind: "recovered", + rowsStripped: 0, + rowsDropped: 3, + rowsDroppedExact: true, + capped: true, + }, + 3 + ) + ).toBe(true); + expect( + landedNothing( + { + kind: "recovered", + rowsStripped: 0, + rowsDropped: 2, + rowsDroppedExact: true, + capped: false, + }, + 3 + ) + ).toBe(false); + }); + + it("is false when the dropped count is only a floor, so a partial drop is never read as total loss", () => { + expect( + landedNothing( + { + kind: "recovered", + rowsStripped: 0, + rowsDropped: 1, + rowsDroppedExact: false, + capped: false, + }, + 1 + ) + ).toBe(false); + }); + + it("is false for a healthy or sanitized insert", () => { + expect(landedNothing({ kind: "inserted", insertResult: {} }, 5)).toBe(false); + expect(landedNothing({ kind: "sanitized", insertResult: {} }, 5)).toBe(false); + }); +}); + +describe("parseStrippableRowNumber", () => { + it("reads the parenthesised position ClickHouse appends", () => { + expect( + parseStrippableRowNumber( + "Cannot parse JSON object here: { ... }: (while reading the value of key attributes): (at row 1942)\n: While executing ParallelParsingBlockInputFormat." + ) + ).toBe(1942); + }); + + it("ignores an `at row N` that appears inside the offending row's own data", () => { + const withDecoyInPayload = + 'Cannot parse JSON object here: {"output":{"dbError":"syntax error at row 7"}}: (while reading the value of key output): (at row 3)'; + + expect(parseStrippableRowNumber(withDecoyInPayload)).toBe(3); + expect(parseRowNumberFromError(withDecoyInPayload)).toBe(7); + }); + + it("takes the last parenthesised position when the payload fakes that form too", () => { + expect( + parseStrippableRowNumber( + 'Cannot parse JSON object here: {"msg":"failed (at row 99)"}: (at row 4)' + ) + ).toBe(4); + }); + + it("returns null when no parenthesised position is present, so the caller bails", () => { + expect(parseStrippableRowNumber("Cannot parse JSON object here: {...}: at row 5")).toBeNull(); + expect(parseStrippableRowNumber("Cannot parse JSON object, no position at all")).toBeNull(); + }); +}); + describe("sanitizeUnknownInPlace", () => { it("returns the string unchanged when it has no surrogates", () => { const result = sanitizeUnknownInPlace("hello world"); @@ -294,3 +430,545 @@ describe("sanitizeRows", () => { expect(rows[1].attributes.bigint).toBe("-100000000000000000000"); }); }); + +describe("insertWithLimitedStrip", () => { + const clean = (id: number): FakeRow => ({ id }); + + it("inserts a healthy batch with no recovery", async () => { + const { landed, insertSync, insertAllowingBadRows, stripJsonColumns, allowCalls } = + makeHarness(); + const rows = [clean(0), clean(1), clean(2)]; + + const outcome = await insertWithLimitedStrip({ + rows, + contextLabel: "test", + logger: silentLogger, + insert: insertSync, + insertSync, + insertAllowingBadRows, + stripJsonColumns, + }); + + expect(outcome.kind).toBe("inserted"); + expect(landed).toHaveLength(3); + expect(allowCalls()).toBe(0); + }); + + it("strips a single poison row and lands the rest in full without bailing", async () => { + const { landed, insertSync, insertAllowingBadRows, stripJsonColumns, allowCalls } = + makeHarness(); + const rows = [clean(0), clean(1), { id: 2, poison: true }, clean(3), clean(4)]; + + const outcome = await insertWithLimitedStrip({ + rows, + contextLabel: "test", + logger: silentLogger, + insert: insertSync, + insertSync, + insertAllowingBadRows, + stripJsonColumns, + }); + + expect(outcome).toEqual({ + kind: "recovered", + rowsStripped: 1, + rowsDropped: 0, + rowsDroppedExact: true, + capped: false, + }); + expect(landed.map((r) => r.id).sort((a, b) => a - b)).toEqual([0, 1, 2, 3, 4]); + expect(landed.find((r) => r.id === 2)?.stripped).toBe(true); + expect(landed.filter((r) => r.id !== 2).every((r) => !r.stripped)).toBe(true); + expect(allowCalls()).toBe(0); + }); + + it("strips up to the limit, then bails to allow_errors and skips the excess poison", async () => { + const { landed, insertSync, insertAllowingBadRows, stripJsonColumns, allowCalls } = + makeHarness(); + const rows = [clean(0), { id: 1, poison: true }, clean(2), { id: 3, poison: true }, clean(4)]; + + const outcome = await insertWithLimitedStrip({ + rows, + contextLabel: "test", + logger: silentLogger, + insert: insertSync, + insertSync, + insertAllowingBadRows, + stripJsonColumns, + maxPoisonStrips: 1, + hasMaterializedViews: false, + }); + + expect(outcome).toEqual({ + kind: "recovered", + rowsStripped: 1, + rowsDropped: 1, + rowsDroppedExact: true, + capped: true, + bailReason: "strip_budget_spent", + }); + expect(allowCalls()).toBe(1); + expect(landed.map((r) => r.id).sort((a, b) => a - b)).toEqual([0, 1, 2, 4]); + expect(landed.find((r) => r.id === 1)?.stripped).toBe(true); + expect(landed.some((r) => r.id === 3)).toBe(false); + }); + + it("strips every poison row when the limit is high enough", async () => { + const { landed, insertSync, insertAllowingBadRows, stripJsonColumns, allowCalls } = + makeHarness(); + const rows = [clean(0), { id: 1, poison: true }, { id: 2, poison: true }, clean(3)]; + + const outcome = await insertWithLimitedStrip({ + rows, + contextLabel: "test", + logger: silentLogger, + insert: insertSync, + insertSync, + insertAllowingBadRows, + stripJsonColumns, + maxPoisonStrips: 3, + }); + + expect(outcome).toEqual({ + kind: "recovered", + rowsStripped: 2, + rowsDropped: 0, + rowsDroppedExact: true, + capped: false, + }); + expect(allowCalls()).toBe(0); + expect(landed.map((r) => r.id).sort((a, b) => a - b)).toEqual([0, 1, 2, 3]); + }); + + it("bails to allow_errors when the failing row cannot be located", async () => { + const landed: FakeRow[] = []; + let allowCalls = 0; + const insertSync = async (rows: FakeRow[]) => { + if (rows.some((r) => r.poison)) { + throw new Error("Cannot parse JSON object here: {...} (no row hint here)"); + } + landed.push(...rows); + return { summary: { written_rows: String(rows.length) } }; + }; + const insertAllowingBadRows = async (rows: FakeRow[]) => { + allowCalls += 1; + const good = rows.filter((r) => !r.poison); + landed.push(...good); + return { summary: { written_rows: String(good.length) } }; + }; + + const outcome = await insertWithLimitedStrip({ + rows: [clean(0), { id: 1, poison: true }, clean(2)], + contextLabel: "test", + logger: silentLogger, + insert: insertSync, + insertSync, + insertAllowingBadRows, + stripJsonColumns: (row) => row, + hasMaterializedViews: false, + }); + + expect(outcome).toEqual({ + kind: "recovered", + rowsStripped: 0, + rowsDropped: 1, + rowsDroppedExact: true, + capped: true, + bailReason: "row_not_locatable", + }); + expect(allowCalls).toBe(1); + expect(landed.map((r) => r.id).sort((a, b) => a - b)).toEqual([0, 2]); + }); + + it("rethrows non-parse errors so the caller's transient-retry path handles them", async () => { + const insert = async () => { + throw new Error("Connection refused"); + }; + + await expect( + insertWithLimitedStrip({ + rows: [clean(0)], + contextLabel: "test", + logger: silentLogger, + insert, + insertSync: insert, + insertAllowingBadRows: insert, + stripJsonColumns: (row) => row, + }) + ).rejects.toThrow("Connection refused"); + }); + + it("strips the row ClickHouse pointed at, not one named by the payload's own text", async () => { + const landed: FakeRow[] = []; + const insertSync = async (rows: FakeRow[]) => { + const badIndex = rows.findIndex((r) => r.poison); + if (badIndex >= 0) { + throw new Error( + `Cannot parse JSON object here: {"output":"failed at row 1"}: (at row ${badIndex + 1})` + ); + } + landed.push(...rows); + return { summary: { written_rows: String(rows.length) } }; + }; + const insertAllowingBadRows = async (rows: FakeRow[]) => { + const good = rows.filter((r) => !r.poison); + landed.push(...good); + return { summary: { written_rows: String(good.length) } }; + }; + + const outcome = await insertWithLimitedStrip({ + rows: [clean(0), clean(1), { id: 2, poison: true }], + contextLabel: "test", + logger: silentLogger, + insert: insertSync, + insertSync, + insertAllowingBadRows, + stripJsonColumns: (row) => ({ ...row, poison: false, stripped: true }), + }); + + expect(outcome).toEqual({ + kind: "recovered", + rowsStripped: 1, + rowsDropped: 0, + rowsDroppedExact: true, + capped: false, + }); + expect(landed.map((r) => r.id).sort((a, b) => a - b)).toEqual([0, 1, 2]); + expect(landed.find((r) => r.id === 2)?.stripped).toBe(true); + expect(landed.filter((r) => r.id !== 2).every((r) => !r.stripped)).toBe(true); + }); + + it("counts the batch as dropped instead of throwing when even allow_errors is rejected", async () => { + const insertSync = async () => { + throw parseErrorAtRow(1); + }; + let allowCalls = 0; + const insertAllowingBadRows = async () => { + allowCalls += 1; + throw new Error("Cannot parse JSON object here: {...}: (at row 1)"); + }; + + const outcome = await insertWithLimitedStrip({ + rows: [clean(0), { id: 1, poison: true }], + contextLabel: "test", + logger: silentLogger, + insert: insertSync, + insertSync, + insertAllowingBadRows, + stripJsonColumns: (row) => row, + }); + + expect(outcome).toEqual({ + kind: "recovered", + rowsStripped: 1, + rowsDropped: 2, + rowsDroppedExact: true, + capped: true, + bailReason: "strip_budget_spent", + }); + expect(allowCalls).toBe(1); + }); + + it("rethrows a transient failure of the allow_errors insert so the retry layer still runs", async () => { + const insertSync = async () => { + throw parseErrorAtRow(1); + }; + const insertAllowingBadRows = async () => { + throw new Error("Connection refused"); + }; + + await expect( + insertWithLimitedStrip({ + rows: [clean(0), { id: 1, poison: true }], + contextLabel: "test", + logger: silentLogger, + insert: insertSync, + insertSync, + insertAllowingBadRows, + stripJsonColumns: (row) => row, + }) + ).rejects.toThrow("Connection refused"); + }); +}); + +/** + * Builds insert doubles for `insertWithBadRowSkip`: a normal insert that fails + * whole when any row is poison, plus an allow-bad-rows insert that lands only + * the good rows and reports the landed count in a ClickHouse-style summary + * (`written_rows`) so the recovery can derive how many rows were skipped. + */ +function makeSkipHarness() { + const landed: FakeRow[] = []; + let insertCalls = 0; + let allowCalls = 0; + const insert = async (rows: FakeRow[]) => { + insertCalls += 1; + const badIndex = rows.findIndex((r) => r.poison || r.unstrippable); + if (badIndex >= 0) throw parseErrorAtRow(badIndex + 1); + landed.push(...rows); + return { summary: { written_rows: String(rows.length) } }; + }; + const insertAllowingBadRows = async (rows: FakeRow[]) => { + allowCalls += 1; + const good = rows.filter((r) => !(r.poison || r.unstrippable)); + landed.push(...good); + return { summary: { written_rows: String(good.length) } }; + }; + return { + landed, + insert, + insertAllowingBadRows, + insertCalls: () => insertCalls, + allowCalls: () => allowCalls, + }; +} + +describe("insertWithBadRowSkip", () => { + const clean = (id: number): FakeRow => ({ id }); + + it("inserts a healthy batch with no recovery", async () => { + const { landed, insert, insertAllowingBadRows, allowCalls } = makeSkipHarness(); + + const outcome = await insertWithBadRowSkip({ + rows: [clean(0), clean(1), clean(2)], + contextLabel: "test", + logger: silentLogger, + insert, + insertAllowingBadRows, + }); + + expect(outcome.kind).toBe("inserted"); + expect(landed).toHaveLength(3); + expect(allowCalls()).toBe(0); + }); + + it("skips the poison rows in one extra insert and counts drops exactly on a table with no materialized views", async () => { + const { landed, insert, insertAllowingBadRows, insertCalls, allowCalls } = makeSkipHarness(); + const rows = [clean(0), { id: 1, poison: true }, clean(2), { id: 3, poison: true }, clean(4)]; + + const outcome = await insertWithBadRowSkip({ + rows, + contextLabel: "test", + logger: silentLogger, + insert, + insertAllowingBadRows, + hasMaterializedViews: false, + }); + + expect(outcome).toEqual({ + kind: "recovered", + rowsStripped: 0, + rowsDropped: 2, + rowsDroppedExact: true, + capped: false, + }); + expect(landed.map((r) => r.id).sort((a, b) => a - b)).toEqual([0, 2, 4]); + expect(insertCalls()).toBe(1); + expect(allowCalls()).toBe(1); + }); + + it("floors a partial drop at one on a table with materialized views, and counts a whole-batch drop exactly", async () => { + const inflatedSummary = (goodCount: number) => ({ + summary: { written_rows: String(goodCount * 3) }, + }); + const insert = async (rows: FakeRow[]) => { + if (rows.some((r) => r.poison)) throw parseErrorAtRow(1); + return inflatedSummary(rows.length); + }; + const insertAllowingBadRows = async (rows: FakeRow[]) => + inflatedSummary(rows.filter((r) => !r.poison).length); + + const partial = await insertWithBadRowSkip({ + rows: [clean(0), { id: 1, poison: true }, clean(2)], + contextLabel: "test", + logger: silentLogger, + insert, + insertAllowingBadRows, + hasMaterializedViews: true, + }); + expect(partial).toEqual({ + kind: "recovered", + rowsStripped: 0, + rowsDropped: 1, + rowsDroppedExact: false, + capped: false, + }); + + const wholeBatch = await insertWithBadRowSkip({ + rows: [ + { id: 0, poison: true }, + { id: 1, poison: true }, + ], + contextLabel: "test", + logger: silentLogger, + insert, + insertAllowingBadRows, + hasMaterializedViews: true, + }); + expect(wholeBatch).toEqual({ + kind: "recovered", + rowsStripped: 0, + rowsDropped: 2, + rowsDroppedExact: true, + capped: false, + }); + }); + + it("sanitizes a repairable batch and lands it in full without skipping any row", async () => { + const landed: FakeRow[] = []; + let allowCalls = 0; + const insert = async (rows: FakeRow[]) => { + if (rows.some((r) => typeof r.msg === "string" && r.msg.includes(HIGH_SURROGATE))) { + throw parseErrorAtRow(1); + } + landed.push(...rows); + return { summary: { written_rows: String(rows.length) } }; + }; + const insertAllowingBadRows = async (rows: FakeRow[]) => { + allowCalls += 1; + landed.push(...rows); + return { summary: { written_rows: String(rows.length) } }; + }; + const rows: FakeRow[] = [ + { id: 0, msg: `bad ${HIGH_SURROGATE}` }, + { id: 1, msg: "clean" }, + ]; + + const outcome = await insertWithBadRowSkip({ + rows, + contextLabel: "test", + logger: silentLogger, + insert, + insertAllowingBadRows, + }); + + expect(outcome.kind).toBe("sanitized"); + expect(allowCalls).toBe(0); + expect(landed).toHaveLength(2); + expect(rows[0].msg).toBe(INVALID_UTF16_SENTINEL); + }); + + it("floors the dropped count at one when the summary has no written_rows count", async () => { + const insert = async (rows: FakeRow[]) => { + if (rows.some((r) => r.poison)) throw parseErrorAtRow(1); + return undefined; + }; + const insertAllowingBadRows = async () => ({ summary: {} }); + + const outcome = await insertWithBadRowSkip({ + rows: [{ id: 0, poison: true }, clean(1)], + contextLabel: "test", + logger: silentLogger, + insert, + insertAllowingBadRows, + }); + + expect(outcome).toEqual({ + kind: "recovered", + rowsStripped: 0, + rowsDropped: 1, + rowsDroppedExact: false, + capped: false, + }); + }); + + it("rethrows non-parse errors so the caller's transient-retry path handles them", async () => { + const insert = async () => { + throw new Error("Connection refused"); + }; + + await expect( + insertWithBadRowSkip({ + rows: [clean(0)], + contextLabel: "test", + logger: silentLogger, + insert, + insertAllowingBadRows: insert, + }) + ).rejects.toThrow("Connection refused"); + }); + + it("counts the batch as dropped instead of throwing when even allow_errors is rejected", async () => { + const insert = async () => { + throw parseErrorAtRow(1); + }; + let allowCalls = 0; + const insertAllowingBadRows = async () => { + allowCalls += 1; + throw new Error("Cannot parse JSON object here: {...}: (at row 1)"); + }; + + const outcome = await insertWithBadRowSkip({ + rows: [clean(0), { id: 1, poison: true }, clean(2)], + contextLabel: "test", + logger: silentLogger, + insert, + insertAllowingBadRows, + }); + + expect(outcome).toEqual({ + kind: "recovered", + rowsStripped: 0, + rowsDropped: 3, + rowsDroppedExact: true, + capped: false, + }); + expect(allowCalls).toBe(1); + }); + + it("rethrows a transient failure of the allow_errors insert so the retry layer still runs", async () => { + const insert = async () => { + throw parseErrorAtRow(1); + }; + const insertAllowingBadRows = async () => { + throw new Error("Connection refused"); + }; + + await expect( + insertWithBadRowSkip({ + rows: [clean(0), { id: 1, poison: true }], + contextLabel: "test", + logger: silentLogger, + insert, + insertAllowingBadRows, + }) + ).rejects.toThrow("Connection refused"); + }); + + it("logs a whole-batch drop at error so alerting fires, and a partial skip only at warn", async () => { + const levels: string[] = []; + const capturingLogger = { + info: () => levels.push("info"), + warn: () => levels.push("warn"), + error: () => levels.push("error"), + }; + + await insertWithBadRowSkip({ + rows: [{ id: 0, poison: true }], + contextLabel: "test", + logger: capturingLogger, + insert: async () => { + throw parseErrorAtRow(1); + }, + insertAllowingBadRows: async () => { + throw new Error("Cannot parse JSON object here: {...}: (at row 1)"); + }, + }); + + expect(levels).toEqual(["error"]); + + levels.length = 0; + const skip = makeSkipHarness(); + + await insertWithBadRowSkip({ + rows: [clean(0), { id: 1, poison: true }], + contextLabel: "test", + logger: capturingLogger, + insert: skip.insert, + insertAllowingBadRows: skip.insertAllowingBadRows, + hasMaterializedViews: false, + }); + + expect(levels).toEqual(["warn"]); + }); +}); diff --git a/internal-packages/clickhouse/src/client/client.ts b/internal-packages/clickhouse/src/client/client.ts index ef18883745b..d6703c863ce 100644 --- a/internal-packages/clickhouse/src/client/client.ts +++ b/internal-packages/clickhouse/src/client/client.ts @@ -747,7 +747,7 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { recordClickhouseError(span, clickhouseError); - return [new InsertError(clickhouseError.message), null]; + return [toInsertError(clickhouseError), null]; } this.logger.debug("Inserted into clickhouse", { @@ -838,7 +838,7 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { }); recordClickhouseError(span, clickhouseError); - return [new InsertError(clickhouseError.message), null]; + return [toInsertError(clickhouseError), null]; } return [null, result]; @@ -900,7 +900,7 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { recordClickhouseError(span, clickhouseError); - return [new InsertError(clickhouseError.message), null]; + return [toInsertError(clickhouseError), null]; } this.logger.debug("Inserted into clickhouse", { @@ -999,7 +999,7 @@ export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter { }); recordClickhouseError(span, clickhouseError); - return [new InsertError(clickhouseError.message), null]; + return [toInsertError(clickhouseError), null]; } this.logger.debug("Inserted into clickhouse", { @@ -1088,6 +1088,11 @@ function classifyClickhouseError( return "fault"; } +function toInsertError(error: Error): InsertError { + const rawMessage = error instanceof ClickHouseError ? error.rawMessage : undefined; + return new InsertError(error.message, { rawMessage }); +} + function recordClickhouseError(span: Span, error: Error): void { if (error instanceof ClickHouseError) { span.setAttributes({ diff --git a/internal-packages/clickhouse/src/client/errors.test.ts b/internal-packages/clickhouse/src/client/errors.test.ts new file mode 100644 index 00000000000..037de63f8d9 --- /dev/null +++ b/internal-packages/clickhouse/src/client/errors.test.ts @@ -0,0 +1,47 @@ +import { ClickHouseError, parseError } from "@clickhouse/client"; +import { describe, expect, it } from "vitest"; +import { InsertError } from "./errors.js"; + +const rawMessage = + 'Code: 117. DB::Exception: Cannot parse JSON object here: {"secret":"customer-payload"}: (at row 3) (INCORRECT_DATA) (version 26.2.1.1)'; + +describe("patched ClickHouseError.rawMessage", () => { + it("carries the untruncated text so the recovery path can read the row hint", () => { + const error = parseError(rawMessage); + + expect(error).toBeInstanceOf(ClickHouseError); + expect((error as ClickHouseError).rawMessage).toContain("at row 3"); + }); + + it("stays out of anything that serializes own enumerable properties", () => { + const error = parseError(rawMessage); + + expect(Object.keys(error)).not.toContain("rawMessage"); + expect(JSON.stringify(error)).not.toContain("customer-payload"); + expect(JSON.stringify({ ...error })).not.toContain("customer-payload"); + }); +}); + +describe("InsertError.rawMessage", () => { + it("is readable by the recovery path", () => { + expect(new InsertError("Cannot parse JSON object here", { rawMessage }).rawMessage).toBe( + rawMessage + ); + }); + + it("stays out of anything that serializes own enumerable properties", () => { + const error = new InsertError("Cannot parse JSON object here", { rawMessage }); + + expect(Object.keys(error)).not.toContain("rawMessage"); + expect(JSON.stringify(error)).not.toContain("customer-payload"); + expect(JSON.stringify({ ...error })).not.toContain("customer-payload"); + expect(error.toString()).not.toContain("customer-payload"); + }); + + it("is absent rather than undefined-valued when no raw message is supplied", () => { + const error = new InsertError("boom"); + + expect(error.rawMessage).toBeUndefined(); + expect(Object.keys(error)).not.toContain("rawMessage"); + }); +}); diff --git a/internal-packages/clickhouse/src/client/errors.ts b/internal-packages/clickhouse/src/client/errors.ts index a620fb54649..906aa87e13d 100644 --- a/internal-packages/clickhouse/src/client/errors.ts +++ b/internal-packages/clickhouse/src/client/errors.ts @@ -24,10 +24,22 @@ export abstract class BaseError<TContext extends ErrorContext = ErrorContext> ex export class InsertError extends BaseError { public readonly retry = true; public readonly name = InsertError.name; - constructor(message: string) { + /** + * Untruncated ClickHouse error text, kept only so the JSON-parse recovery path + * can read the `(at row N)` hint that `message` drops. Defined non-enumerable + * on purpose: ClickHouse embeds a snippet of the offending row in its parse + * errors, so this must not reach structured logs or error reporting, both of + * which serialize own enumerable properties. Direct reads still work. + */ + declare readonly rawMessage?: string; + constructor(message: string, options?: { rawMessage?: string }) { super({ message, }); + Object.defineProperty(this, "rawMessage", { + value: options?.rawMessage, + enumerable: false, + }); } } export class QueryError extends BaseError<{ query: string }> { diff --git a/package.json b/package.json index f5987056204..2708b06be73 100644 --- a/package.json +++ b/package.json @@ -91,7 +91,8 @@ "@window-splitter/state@1.1.3": "patches/@window-splitter__state@1.1.3.patch", "streamdown@2.5.0": "patches/streamdown@2.5.0.patch", "tsup@8.4.0": "patches/tsup@8.4.0.patch", - "@remix-run/router@1.23.3": "patches/@remix-run__router@1.23.3.patch" + "@remix-run/router@1.23.3": "patches/@remix-run__router@1.23.3.patch", + "@clickhouse/client-common@1.12.1": "patches/@clickhouse__client-common@1.12.1.patch" }, "overrides": { "typescript": "catalog:", diff --git a/patches/@clickhouse__client-common@1.12.1.patch b/patches/@clickhouse__client-common@1.12.1.patch new file mode 100644 index 00000000000..b5e0ac03add --- /dev/null +++ b/patches/@clickhouse__client-common@1.12.1.patch @@ -0,0 +1,29 @@ +diff --git a/dist/error/error.d.ts b/dist/error/error.d.ts +index fb04ebb65a8c7115c465093462edaafa14a814de..30b370bf2eb8ba7491c249db96d565a67fe367c6 100644 +--- a/dist/error/error.d.ts ++++ b/dist/error/error.d.ts +@@ -7,6 +7,7 @@ interface ParsedClickHouseError { + export declare class ClickHouseError extends Error { + readonly code: string; + readonly type: string | undefined; ++ rawMessage?: string; + constructor({ message, code, type }: ParsedClickHouseError); + } + export declare function parseError(input: string | Error): ClickHouseError | Error; +diff --git a/dist/error/error.js b/dist/error/error.js +index e06fa4ab36537c2a448857c1b001e70cd771bfe5..9750bd7e1285ac8cf2c8095b592ed344126176b6 100644 +--- a/dist/error/error.js ++++ b/dist/error/error.js +@@ -35,7 +35,11 @@ function parseError(input) { + const match = message.match(errorRe); + const groups = match?.groups; + if (groups) { +- return new ClickHouseError(groups); ++ const error = new ClickHouseError(groups); ++ if (!inputIsError && groups.message.startsWith("Cannot parse JSON object")) { ++ Object.defineProperty(error, "rawMessage", { value: message, enumerable: false }); ++ } ++ return error; + } + else { + return inputIsError ? input : new Error(input); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dce3aa6e582..6c8be5550d8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -58,6 +58,9 @@ patchedDependencies: '@changesets/assemble-release-plan@5.2.4': hash: a7a12643e8d89a00d5322f750c292e8567b5ee0fc9c613a238a1184c25533e4b path: patches/@changesets__assemble-release-plan@5.2.4.patch + '@clickhouse/client-common@1.12.1': + hash: 784ccad1302960bef63d3f9df9085d5d2fb748593d47030ee00a937e945f3c9d + path: patches/@clickhouse__client-common@1.12.1.patch '@kubernetes/client-node@1.0.0': hash: ba1a06f46256cdb8d6faf7167246692c0de2e7cd846a9dc0f13be0137e1c3745 path: patches/@kubernetes__client-node@1.0.0.patch @@ -17728,7 +17731,7 @@ snapshots: '@clickhouse/client-common@1.11.1': {} - '@clickhouse/client-common@1.12.1': {} + '@clickhouse/client-common@1.12.1(patch_hash=784ccad1302960bef63d3f9df9085d5d2fb748593d47030ee00a937e945f3c9d)': {} '@clickhouse/client@1.11.1': dependencies: @@ -17736,7 +17739,7 @@ snapshots: '@clickhouse/client@1.12.1': dependencies: - '@clickhouse/client-common': 1.12.1 + '@clickhouse/client-common': 1.12.1(patch_hash=784ccad1302960bef63d3f9df9085d5d2fb748593d47030ee00a937e945f3c9d) '@codemirror/autocomplete@6.4.0(@codemirror/language@6.3.2)(@codemirror/state@6.2.0)(@codemirror/view@6.7.2)(@lezer/common@1.3.0)': dependencies: From f9c8d518c7da654104fa343dca167db50428db47 Mon Sep 17 00:00:00 2001 From: Eric Allam <eric@trigger.dev> Date: Sat, 1 Aug 2026 11:32:21 +0100 Subject: [PATCH 53/67] perf(webapp,run-engine,database): resolve the newest worker and deployment by createdAt (#4452) --- .../worker-deployment-lookup-ordering.md | 6 + .../clickhouseRunsRepository.server.ts | 3 - .../app/v3/models/workerDeployment.server.ts | 6 +- .../controlPlaneResolver.server.ts | 5 +- .../controlPlaneResolver.server.test.ts | 111 +++++++++++++- .../migration.sql | 1 + .../database/prisma/schema.prisma | 1 + .../src/engine/controlPlaneResolver.ts | 11 +- ...dequeueSystem.controlPlaneResolver.test.ts | 139 ++++++++++++++++++ 9 files changed, 262 insertions(+), 21 deletions(-) create mode 100644 .server-changes/worker-deployment-lookup-ordering.md create mode 100644 internal-packages/database/prisma/migrations/20260731160000_add_worker_deployment_environment_id_created_at_index/migration.sql diff --git a/.server-changes/worker-deployment-lookup-ordering.md b/.server-changes/worker-deployment-lookup-ordering.md new file mode 100644 index 00000000000..1dbfbdd253a --- /dev/null +++ b/.server-changes/worker-deployment-lookup-ordering.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Speeds up resolving the latest worker version and deployment for an environment, removing an occasional stall when triggering runs in projects that have accumulated many deployed versions. diff --git a/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts b/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts index 2aa9d5f3a68..c9fefd1da10 100644 --- a/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts +++ b/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts @@ -271,9 +271,6 @@ export class ClickHouseRunsRepository implements IRunsRepository { in: ids, }, }, - orderBy: { - id: "desc", - }, select: { id: true, friendlyId: true, diff --git a/apps/webapp/app/v3/models/workerDeployment.server.ts b/apps/webapp/app/v3/models/workerDeployment.server.ts index 5934cd07620..48995bf2b29 100644 --- a/apps/webapp/app/v3/models/workerDeployment.server.ts +++ b/apps/webapp/app/v3/models/workerDeployment.server.ts @@ -128,14 +128,12 @@ export async function findCurrentWorkerDeployment({ } // We need to get the latest deployment of the given type - const latestDeployment = await prisma.workerDeployment.findFirst({ + const latestDeployment = await $prisma.workerDeployment.findFirst({ where: { environmentId, type, }, - orderBy: { - id: "desc", - }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], select: { id: true, imageReference: true, diff --git a/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts b/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts index d6a706c9665..eb19a7fb6c1 100644 --- a/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts @@ -432,7 +432,7 @@ export class ControlPlaneResolver { // MANAGED deployment. const latestV2Deployment = await client.workerDeployment.findFirst({ where: { environmentId, type: "MANAGED" }, - orderBy: { id: "desc" }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], include: { worker: { include: { tasks: true, queues: true } } }, }); @@ -455,7 +455,6 @@ export class ControlPlaneResolver { const worker = await client.backgroundWorker.findFirst({ where: { id: workerId }, include: { deployment: true, tasks: true, queues: true }, - orderBy: { id: "desc" }, }); if (!worker) { @@ -472,7 +471,7 @@ export class ControlPlaneResolver { const worker = await client.backgroundWorker.findFirst({ where: { runtimeEnvironmentId: environmentId }, include: { tasks: true, queues: true }, - orderBy: { id: "desc" }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], }); if (!worker) { diff --git a/apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts b/apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts index e85843114db..af734ca1f75 100644 --- a/apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts +++ b/apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts @@ -62,7 +62,12 @@ async function seedControlPlane(prisma: PrismaClient) { async function seedWorker( prisma: PrismaClient, ctx: { projectId: string; environmentId: string }, - opts?: { promote?: boolean } + opts?: { + promote?: boolean; + createdAt?: Date; + deploymentType?: "MANAGED" | "UNMANAGED" | "V1"; + deploymentCreatedAt?: Date; + } ) { const n = seedCounter++; const worker = await prisma.backgroundWorker.create({ @@ -74,6 +79,7 @@ async function seedWorker( version: `2024.1.${n}`, metadata: {}, engine: "V2", + ...(opts?.createdAt ? { createdAt: opts.createdAt } : {}), }, }); const task = await prisma.backgroundWorkerTask.create({ @@ -104,11 +110,12 @@ async function seedWorker( contentHash: `hash_${n}`, version: worker.version, shortCode: `dep_${n}`, - type: "MANAGED", + type: opts?.deploymentType ?? "MANAGED", status: "DEPLOYED", projectId: ctx.projectId, environmentId: ctx.environmentId, workerId: worker.id, + ...(opts?.deploymentCreatedAt ? { createdAt: opts.deploymentCreatedAt } : {}), }, }); await prisma.workerDeploymentPromotion.create({ @@ -748,3 +755,103 @@ heteroPostgresTest( expect(reads()).toBe(readsAfterFirst * 2); } ); + +heteroPostgresTest( + "resolveWorkerVersion (DEVELOPMENT) resolves the newest worker by createdAt, not by id", + async ({ prisma14 }) => { + const { environment, project } = await seedControlPlane(prisma14); + const ctx = { projectId: project.id, environmentId: environment.id }; + + const newest = await seedWorker(prisma14, ctx, { + createdAt: new Date("2026-07-31T12:00:00.000Z"), + }); + const oldest = await seedWorker(prisma14, ctx, { + createdAt: new Date("2026-07-30T12:00:00.000Z"), + }); + + expect(oldest.worker.id > newest.worker.id).toBe(true); + expect(oldest.worker.createdAt < newest.worker.createdAt).toBe(true); + + const resolver = new ControlPlaneResolver({ + controlPlaneReplica: prisma14, + controlPlanePrimary: prisma14, + cache: new ControlPlaneCache(), + splitEnabled: () => true, + }); + + const resolved = await resolver.resolveWorkerVersion({ + environmentId: environment.id, + type: "DEVELOPMENT", + }); + + expect(resolved).not.toBeNull(); + expect(resolved!.worker.id).toBe(newest.worker.id); + } +); + +heteroPostgresTest( + "resolveWorkerVersion latest-MANAGED fallback resolves by createdAt, not by id", + async ({ prisma14 }) => { + const { environment, project } = await seedControlPlane(prisma14); + const ctx = { projectId: project.id, environmentId: environment.id }; + + await seedWorker(prisma14, ctx, { promote: true, deploymentType: "V1" }); + + const newest = await seedWorker(prisma14, ctx, { + promote: false, + deploymentCreatedAt: new Date("2026-07-31T12:00:00.000Z"), + }); + const oldest = await seedWorker(prisma14, ctx, { + promote: false, + deploymentCreatedAt: new Date("2026-07-30T12:00:00.000Z"), + }); + + const newestDeployment = await prisma14.workerDeployment.create({ + data: { + friendlyId: `deployment_newest_${environment.id}`, + contentHash: "hash_newest", + version: newest.worker.version, + shortCode: "dep_newest", + type: "MANAGED", + status: "DEPLOYED", + projectId: project.id, + environmentId: environment.id, + workerId: newest.worker.id, + createdAt: new Date("2026-07-31T12:00:00.000Z"), + }, + }); + const oldestDeployment = await prisma14.workerDeployment.create({ + data: { + friendlyId: `deployment_oldest_${environment.id}`, + contentHash: "hash_oldest", + version: oldest.worker.version, + shortCode: "dep_oldest", + type: "MANAGED", + status: "DEPLOYED", + projectId: project.id, + environmentId: environment.id, + workerId: oldest.worker.id, + createdAt: new Date("2026-07-30T12:00:00.000Z"), + }, + }); + + expect(oldestDeployment.id > newestDeployment.id).toBe(true); + expect(oldestDeployment.createdAt < newestDeployment.createdAt).toBe(true); + + const resolver = new ControlPlaneResolver({ + controlPlaneReplica: prisma14, + controlPlanePrimary: prisma14, + cache: new ControlPlaneCache(), + splitEnabled: () => true, + }); + + const resolved = await resolver.resolveWorkerVersion({ + environmentId: environment.id, + type: "PRODUCTION", + }); + + expect(resolved).not.toBeNull(); + expect(resolved!.deployment!.id).toBe(newestDeployment.id); + expect(resolved!.worker.id).toBe(newest.worker.id); + } +); diff --git a/internal-packages/database/prisma/migrations/20260731160000_add_worker_deployment_environment_id_created_at_index/migration.sql b/internal-packages/database/prisma/migrations/20260731160000_add_worker_deployment_environment_id_created_at_index/migration.sql new file mode 100644 index 00000000000..57d9eec96c4 --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260731160000_add_worker_deployment_environment_id_created_at_index/migration.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS "WorkerDeployment_environmentId_createdAt_idx" ON "public"."WorkerDeployment"("environmentId", "createdAt"); diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 74c6d72a9c5..ca1d868ab04 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2208,6 +2208,7 @@ model WorkerDeployment { @@unique([projectId, shortCode]) @@unique([environmentId, version]) @@index([commitSHA]) + @@index([environmentId, createdAt]) } enum WorkerDeploymentStatus { diff --git a/internal-packages/run-engine/src/engine/controlPlaneResolver.ts b/internal-packages/run-engine/src/engine/controlPlaneResolver.ts index 89b0e15c48c..da5f7b00603 100644 --- a/internal-packages/run-engine/src/engine/controlPlaneResolver.ts +++ b/internal-packages/run-engine/src/engine/controlPlaneResolver.ts @@ -234,9 +234,7 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver { tasks: true, queues: true, }, - orderBy: { - id: "desc", - }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], }); if (!worker) { @@ -256,9 +254,6 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver { tasks: true, queues: true, }, - orderBy: { - id: "desc", - }, }); if (!worker) { @@ -315,9 +310,7 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver { environmentId, type: "MANAGED", }, - orderBy: { - id: "desc", - }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], include: { worker: { include: { diff --git a/internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts b/internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts index 5ede5c89db2..3ea56bde8a6 100644 --- a/internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts +++ b/internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts @@ -538,3 +538,142 @@ describe("DequeueSystem controlPlaneResolver (single-DB passthrough)", () => { } ); }); + +heteroPostgresTest( + "resolveWorkerVersion (DEVELOPMENT) resolves the newest worker by createdAt, not by id", + async ({ prisma14 }) => { + const cp = await seedControlPlane( + prisma14 as unknown as PrismaClient, + "cpord", + "ordering-task" + ); + + await prisma14.backgroundWorker.update({ + where: { id: cp.worker.id }, + data: { createdAt: new Date("2026-07-01T12:00:00.000Z") }, + }); + + const newest = await prisma14.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + contentHash: "hash_newest", + projectId: cp.project.id, + runtimeEnvironmentId: cp.environment.id, + version: "20260731.1", + metadata: {}, + engine: "V2", + createdAt: new Date("2026-07-31T12:00:00.000Z"), + }, + }); + const oldest = await prisma14.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + contentHash: "hash_oldest", + projectId: cp.project.id, + runtimeEnvironmentId: cp.environment.id, + version: "20260730.1", + metadata: {}, + engine: "V2", + createdAt: new Date("2026-07-30T12:00:00.000Z"), + }, + }); + + expect(oldest.id > newest.id).toBe(true); + expect(oldest.createdAt < newest.createdAt).toBe(true); + + const resolver = new PassthroughControlPlaneResolver({ + prisma: prisma14 as unknown as PrismaClient, + }); + + const resolved = await resolver.resolveWorkerVersion({ + environmentId: cp.environment.id, + type: "DEVELOPMENT", + }); + + assertNonNullable(resolved); + expect(resolved.worker.id).toBe(newest.id); + } +); + +heteroPostgresTest( + "resolveWorkerVersion latest-MANAGED fallback resolves by createdAt, not by id", + async ({ prisma14 }) => { + const cp = await seedControlPlane( + prisma14 as unknown as PrismaClient, + "cpfall", + "fallback-task" + ); + + await prisma14.workerDeployment.update({ + where: { id: cp.deployment.id }, + data: { type: "V1" }, + }); + + const newestWorker = await prisma14.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + contentHash: "hash_newest", + projectId: cp.project.id, + runtimeEnvironmentId: cp.environment.id, + version: "20260731.1", + metadata: {}, + engine: "V2", + }, + }); + const oldestWorker = await prisma14.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + contentHash: "hash_oldest", + projectId: cp.project.id, + runtimeEnvironmentId: cp.environment.id, + version: "20260730.1", + metadata: {}, + engine: "V2", + }, + }); + + const newestDeployment = await prisma14.workerDeployment.create({ + data: { + friendlyId: generateFriendlyId("deployment"), + contentHash: "hash_newest", + version: "20260731.1", + shortCode: "short_code_newest", + status: "DEPLOYED", + projectId: cp.project.id, + environmentId: cp.environment.id, + workerId: newestWorker.id, + type: "MANAGED", + createdAt: new Date("2026-07-31T12:00:00.000Z"), + }, + }); + const oldestDeployment = await prisma14.workerDeployment.create({ + data: { + friendlyId: generateFriendlyId("deployment"), + contentHash: "hash_oldest", + version: "20260730.1", + shortCode: "short_code_oldest", + status: "DEPLOYED", + projectId: cp.project.id, + environmentId: cp.environment.id, + workerId: oldestWorker.id, + type: "MANAGED", + createdAt: new Date("2026-07-30T12:00:00.000Z"), + }, + }); + + expect(oldestDeployment.id > newestDeployment.id).toBe(true); + expect(oldestDeployment.createdAt < newestDeployment.createdAt).toBe(true); + + const resolver = new PassthroughControlPlaneResolver({ + prisma: prisma14 as unknown as PrismaClient, + }); + + const resolved = await resolver.resolveWorkerVersion({ + environmentId: cp.environment.id, + type: "PRODUCTION", + }); + + assertNonNullable(resolved); + expect(resolved.deployment?.id).toBe(newestDeployment.id); + } +); From db6228dd1e0d2e4cd4a2cdc2ddf113cfb8c305d6 Mon Sep 17 00:00:00 2001 From: Eric Allam <eric@trigger.dev> Date: Sat, 1 Aug 2026 11:33:34 +0100 Subject: [PATCH 54/67] chore(webapp,core,sdk): upgrade @s2-dev/streamstore to 0.25 and migrate S2 hosts (#4349) --- .changeset/chat-agent-preload-oom-recovery.md | 5 + .changeset/chat-handover-cursor-off-by-one.md | 5 + .changeset/chat-session-caught-up-resume.md | 7 + .github/workflows/e2e-webapp.yml | 7 +- apps/webapp/app/env.server.ts | 2 + .../realtime/s2realtimeStreams.server.ts | 23 +- .../realtime/streamBasinProvisioner.server.ts | 4 +- .../realtime/v1StreamsGlobal.server.ts | 2 + apps/webapp/package.json | 3 +- apps/webapp/test/helpers/agentHarness.ts | 197 +++ apps/webapp/test/helpers/sessionStream.ts | 280 +++ apps/webapp/test/helpers/testChatAgent.ts | 341 ++++ apps/webapp/test/session-agent.e2e.test.ts | 1536 +++++++++++++++++ .../test/session-stream.browser.e2e.test.ts | 133 ++ apps/webapp/test/session-stream.e2e.test.ts | 326 ++++ apps/webapp/vite.config.ts | 1 + apps/webapp/vitest.e2e.config.ts | 7 + docs/ai-chat/client-protocol.mdx | 6 +- internal-packages/testcontainers/src/s2.ts | 67 + .../testcontainers/src/webapp.ts | 97 +- packages/cli-v3/package.json | 2 +- packages/core/package.json | 2 +- .../core/src/v3/apiClient/runStream.test.ts | 40 + packages/core/src/v3/apiClient/runStream.ts | 94 +- .../src/v3/realtimeStreams/streamsWriterV2.ts | 14 +- packages/core/src/v3/test/index.ts | 7 + .../core/src/v3/test/mock-task-context.ts | 37 +- .../src/v3/test/session-waitpoint-backend.ts | 267 +++ packages/trigger-sdk/src/v3/ai.ts | 17 +- packages/trigger-sdk/src/v3/chat-client.ts | 23 +- packages/trigger-sdk/src/v3/chat.ts | 5 +- pnpm-lock.yaml | 21 +- 32 files changed, 3521 insertions(+), 57 deletions(-) create mode 100644 .changeset/chat-agent-preload-oom-recovery.md create mode 100644 .changeset/chat-handover-cursor-off-by-one.md create mode 100644 .changeset/chat-session-caught-up-resume.md create mode 100644 apps/webapp/test/helpers/agentHarness.ts create mode 100644 apps/webapp/test/helpers/sessionStream.ts create mode 100644 apps/webapp/test/helpers/testChatAgent.ts create mode 100644 apps/webapp/test/session-agent.e2e.test.ts create mode 100644 apps/webapp/test/session-stream.browser.e2e.test.ts create mode 100644 apps/webapp/test/session-stream.e2e.test.ts create mode 100644 internal-packages/testcontainers/src/s2.ts create mode 100644 packages/core/src/v3/test/session-waitpoint-backend.ts diff --git a/.changeset/chat-agent-preload-oom-recovery.md b/.changeset/chat-agent-preload-oom-recovery.md new file mode 100644 index 00000000000..7885a3530ab --- /dev/null +++ b/.changeset/chat-agent-preload-oom-recovery.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/sdk": patch +--- + +Fix a preloaded `chat.agent` run dropping an in-flight message when it retries after an out-of-memory error. The message being processed when the run hit the OOM is now recovered and re-run on the retry, instead of being skipped while the run waited for a new message. diff --git a/.changeset/chat-handover-cursor-off-by-one.md b/.changeset/chat-handover-cursor-off-by-one.md new file mode 100644 index 00000000000..6829d113d7b --- /dev/null +++ b/.changeset/chat-handover-cursor-off-by-one.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Fix a chunk occasionally dropped when a chat.agent run takes over from the warm first turn. The realtime stream writer now reports the inclusive last-written position as the resume cursor, so the agent's first record after the handover is no longer skipped. diff --git a/.changeset/chat-session-caught-up-resume.md b/.changeset/chat-session-caught-up-resume.md new file mode 100644 index 00000000000..b405bf60472 --- /dev/null +++ b/.changeset/chat-session-caught-up-resume.md @@ -0,0 +1,7 @@ +--- +"@trigger.dev/core": patch +"@trigger.dev/sdk": patch +"trigger.dev": patch +--- + +`AgentChat.reconnect()` now settles promptly when reconnecting to an idle chat instead of holding the connection open for the full long-poll window. Also upgrades the S2 streamstore client to 0.25 and moves realtime streams to S2's current hosts. diff --git a/.github/workflows/e2e-webapp.yml b/.github/workflows/e2e-webapp.yml index 1931cfdd336..1c1525f31bc 100644 --- a/.github/workflows/e2e-webapp.yml +++ b/.github/workflows/e2e-webapp.yml @@ -15,7 +15,7 @@ jobs: e2eTests: name: "🧪 E2E Tests: Webapp" runs-on: warp-ubuntu-latest-x64-16x - timeout-minutes: 20 + timeout-minutes: 30 env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} steps: @@ -80,6 +80,8 @@ jobs: docker pull postgres:14 docker pull redis:7.2 docker pull testcontainers/ryuk:0.11.0 + docker pull ghcr.io/s2-streamstore/s2:0.40.0@sha256:b26249e2ede0949755f5af8028185dc2bcfc3aa2db21eb9610543d144eb6ee9d + docker pull minio/minio:latest echo "Image pre-pull complete" - name: 📥 Download deps @@ -91,6 +93,9 @@ jobs: - name: 🏗️ Build Webapp run: pnpm run build --filter webapp + - name: 🎭 Install Playwright Chromium + run: cd apps/webapp && pnpm exec playwright install chromium + - name: 🧪 Run Webapp E2E Tests run: cd apps/webapp && pnpm exec vitest run --config vitest.e2e.config.ts --reporter=default env: diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 9e0a9fa84ed..541ff36fa6e 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -2122,6 +2122,8 @@ const EnvironmentSchema = z REALTIME_STREAMS_S2_BASIN: z.string().optional(), REALTIME_STREAMS_S2_ACCESS_TOKEN: z.string().optional(), REALTIME_STREAMS_S2_ENDPOINT: z.string().optional(), + REALTIME_STREAMS_S2_ACCOUNT_URL: z.string().default("https://a.s2.dev/v1"), + REALTIME_STREAMS_S2_BASIN_URL: z.string().default("https://{basin}.b.s2.dev/v1"), REALTIME_STREAMS_S2_SKIP_ACCESS_TOKENS: z.enum(["true", "false"]).default("false"), REALTIME_STREAMS_S2_ACCESS_TOKEN_EXPIRATION_IN_MS: z.coerce .number() diff --git a/apps/webapp/app/services/realtime/s2realtimeStreams.server.ts b/apps/webapp/app/services/realtime/s2realtimeStreams.server.ts index 787b3f1ab44..aff543d3052 100644 --- a/apps/webapp/app/services/realtime/s2realtimeStreams.server.ts +++ b/apps/webapp/app/services/realtime/s2realtimeStreams.server.ts @@ -44,6 +44,10 @@ export type S2RealtimeStreamsOptions = { // Custom endpoint for s2-lite (self-hosted) endpoint?: string; // e.g., "http://localhost:4566/v1" + /** Account-level API base for account/basin ops. Defaults to S2 cloud. */ + accountUrl?: string; + /** Per-basin API base, with a `{basin}` placeholder. Defaults to S2 cloud. */ + basinUrl?: string; // Skip access token issuance (s2-lite doesn't support /access-tokens) skipAccessTokens?: boolean; @@ -74,6 +78,15 @@ export type S2RealtimeStreamsOptions = { const S2_TOKEN_OPS = ["append", "create-stream", "trim"] as const; const S2_TOKEN_OPS_FINGERPRINT = [...S2_TOKEN_OPS].sort().join(","); +/** + * Placeholder handed back as the S2 access token when `skipAccessTokens` is set + * and no token is configured (self-hosted s2-lite ignores the token entirely). + * The SDK's session-stream writer rejects an empty access token as "no S2 + * credentials" and never opens the writer, so the token must be non-empty even + * when it is semantically unused. + */ +const SKIP_ACCESS_TOKENS_SENTINEL = "s2-skip-access-tokens"; + type S2IssueAccessTokenResponse = { access_token: string }; type S2AppendInput = { records: { body: string }[] }; type S2AppendAck = { @@ -107,8 +120,10 @@ export class S2RealtimeStreams implements StreamResponder, StreamIngestor { constructor(opts: S2RealtimeStreamsOptions) { this.basin = opts.basin; - this.baseUrl = opts.endpoint ?? `https://${this.basin}.b.aws.s2.dev/v1`; - this.accountUrl = opts.endpoint ?? `https://aws.s2.dev/v1`; + this.baseUrl = + opts.endpoint ?? + (opts.basinUrl ?? `https://{basin}.b.s2.dev/v1`).replace("{basin}", this.basin); + this.accountUrl = opts.endpoint ?? opts.accountUrl ?? `https://a.s2.dev/v1`; this.endpoint = opts.endpoint; this.token = opts.accessToken; this.streamPrefix = opts.streamPrefix ?? ""; @@ -168,7 +183,7 @@ export class S2RealtimeStreams implements StreamResponder, StreamIngestor { relativeName: string ): Promise<{ responseHeaders?: Record<string, string> }> { const accessToken = this.skipAccessTokens - ? this.token + ? this.token || SKIP_ACCESS_TOKENS_SENTINEL : await this.getS2AccessToken(randomUUID()); return { @@ -178,7 +193,7 @@ export class S2RealtimeStreams implements StreamResponder, StreamIngestor { "X-S2-Basin": this.basin, "X-S2-Flush-Interval-Ms": this.flushIntervalMs.toString(), "X-S2-Max-Retries": this.maxRetries.toString(), - ...(this.endpoint ? { "X-S2-Endpoint": this.endpoint } : {}), + "X-S2-Endpoint": this.baseUrl, }, }; } diff --git a/apps/webapp/app/services/realtime/streamBasinProvisioner.server.ts b/apps/webapp/app/services/realtime/streamBasinProvisioner.server.ts index 98316b84fe8..92c07104a8f 100644 --- a/apps/webapp/app/services/realtime/streamBasinProvisioner.server.ts +++ b/apps/webapp/app/services/realtime/streamBasinProvisioner.server.ts @@ -185,7 +185,7 @@ type CreateBasinOptions = { }; async function s2CreateBasin(name: string, opts: CreateBasinOptions): Promise<void> { - const url = `https://aws.s2.dev/v1/basins`; + const url = `${env.REALTIME_STREAMS_S2_ACCOUNT_URL}/basins`; const body = { basin: name, config: { @@ -222,7 +222,7 @@ type ReconfigureBasinOptions = { }; async function s2ReconfigureBasin(name: string, opts: ReconfigureBasinOptions): Promise<void> { - const url = `https://aws.s2.dev/v1/basins/${encodeURIComponent(name)}`; + const url = `${env.REALTIME_STREAMS_S2_ACCOUNT_URL}/basins/${encodeURIComponent(name)}`; const body = { default_stream_config: { retention_policy: { age: parseDuration(opts.retentionPolicy) }, diff --git a/apps/webapp/app/services/realtime/v1StreamsGlobal.server.ts b/apps/webapp/app/services/realtime/v1StreamsGlobal.server.ts index 4c8b8b9f67f..a6801b8a1b0 100644 --- a/apps/webapp/app/services/realtime/v1StreamsGlobal.server.ts +++ b/apps/webapp/app/services/realtime/v1StreamsGlobal.server.ts @@ -70,6 +70,8 @@ export function getRealtimeStreamInstance( basin: resolvedBasin, accessToken: env.REALTIME_STREAMS_S2_ACCESS_TOKEN ?? "", endpoint: env.REALTIME_STREAMS_S2_ENDPOINT, + accountUrl: env.REALTIME_STREAMS_S2_ACCOUNT_URL, + basinUrl: env.REALTIME_STREAMS_S2_BASIN_URL, skipAccessTokens: env.REALTIME_STREAMS_S2_SKIP_ACCESS_TOKENS === "true", streamPrefix: streamPrefixFor(environment, resolvedBasin), logLevel: env.REALTIME_STREAMS_S2_LOG_LEVEL, diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 06731a53e1c..849db7fc773 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -104,7 +104,7 @@ "@remix-run/react": "2.17.5", "@remix-run/router": "^1.23.3", "@remix-run/server-runtime": "2.17.5", - "@s2-dev/streamstore": "^0.22.10", + "@s2-dev/streamstore": "^0.25.0", "@sentry/remix": "9.46.0", "@slack/web-api": "7.16.0", "@socket.io/redis-adapter": "^8.3.0", @@ -222,6 +222,7 @@ "@internal/clickhouse": "workspace:*", "@internal/replication": "workspace:*", "@internal/testcontainers": "workspace:*", + "@playwright/test": "^1.36.2", "@remix-run/dev": "2.17.5", "@remix-run/testing": "^2.17.5", "@sentry/cli": "2.50.2", diff --git a/apps/webapp/test/helpers/agentHarness.ts b/apps/webapp/test/helpers/agentHarness.ts new file mode 100644 index 00000000000..81bf928b011 --- /dev/null +++ b/apps/webapp/test/helpers/agentHarness.ts @@ -0,0 +1,197 @@ +import { apiClientManager, resourceCatalog } from "@trigger.dev/core/v3"; +import type { LocalsKey } from "@trigger.dev/core/v3"; +import type { LanguageModel } from "ai"; +import { + installSessionWaitpointBackend, + runInMockTaskContext, + StandardSessionStreamManager, +} from "@trigger.dev/core/v3/test"; + +export type RunRealChatAgentOptions = { + agentId: string; + baseUrl: string; + addressingKey: string; + /** + * The environment secret key. The agent writes `.out` and reads `.in` as the + * backend (PRIVATE auth) — the `.out` channel rejects client session tokens. + */ + secretKey: string; + model: LanguageModel; + modelLocal: LocalsKey<LanguageModel>; + runId?: string; + /** + * Boot as a continuation of a previous run for the same session. Gates the + * snapshot + `.out`/`.in` replay boot path, so the agent restores prior + * history instead of treating the chat as brand new. + */ + continuation?: boolean; + previousRunId?: string; + /** + * Idle window (seconds) before the turn loop falls through from the SSE + * once() to the suspending `session.in.wait()`. Set this low to force the + * suspend/resume path in a test. + */ + idleTimeoutInSeconds?: number; + /** + * `ctx.attempt.number`. A value greater than 1 makes the boot treat the run + * as a retry (`couldHavePriorState`), restoring from the snapshot + `.in` + * replay. Used to model an OOM retry re-dispatch. + */ + attemptNumber?: number; +}; + +export type RunningAgent = { + done: Promise<void>; + close: () => Promise<void>; +}; + +/** + * Run the real `chat.agent` turn loop in-process, wired to a running webapp: + * `apiClientManager` + a real `StandardSessionStreamManager` point the agent's + * `.in`/`.out` at the webapp's Session streams (real S2 + SSE), the model is + * injected via locals (so it survives without serialization), and turns are + * driven by appending to `.in` over HTTP. Callers keep each message inside the + * idle window and `close()` promptly so the run-engine suspend path is never + * reached. + */ +export function runRealChatAgent(opts: RunRealChatAgentOptions): RunningAgent { + apiClientManager.setGlobalAPIClientConfiguration({ + baseURL: opts.baseUrl, + accessToken: opts.secretKey, + }); + const apiClient = apiClientManager.clientOrThrow(); + const manager = new StandardSessionStreamManager(apiClient, opts.baseUrl); + const { backend, runtimeManager, restore } = installSessionWaitpointBackend(apiClient); + + const taskEntry = resourceCatalog.getTask(opts.agentId); + if (!taskEntry) { + restore(); + throw new Error(`runRealChatAgent: agent "${opts.agentId}" is not registered`); + } + const runFn = taskEntry.fns.run as ( + payload: unknown, + params: { ctx: unknown; signal: AbortSignal } + ) => Promise<unknown>; + + const runSignal = new AbortController(); + runSignal.signal.addEventListener("abort", () => { + try { + backend.disable(); + } catch {} + }); + const runId = opts.runId ?? `run_${opts.addressingKey}`; + + const idle = + opts.idleTimeoutInSeconds !== undefined + ? { idleTimeoutInSeconds: opts.idleTimeoutInSeconds } + : {}; + + const done = ( + runInMockTaskContext( + async (drivers) => { + drivers.locals.set(opts.modelLocal, opts.model); + const payload = opts.continuation + ? { + chatId: opts.addressingKey, + continuation: true, + metadata: {}, + ...idle, + ...(opts.previousRunId ? { previousRunId: opts.previousRunId } : {}), + } + : { chatId: opts.addressingKey, trigger: "preload", metadata: {}, ...idle }; + await runFn(payload, { ctx: drivers.ctx, signal: runSignal.signal }); + }, + { + ctx: { + run: { id: runId }, + ...(opts.attemptNumber !== undefined ? { attempt: { number: opts.attemptNumber } } : {}), + }, + sessionStreamManager: manager, + runtimeManager, + } + ) as Promise<void> + ).finally(restore); + + return { + done, + close: async () => { + try { + await fetch( + `${opts.baseUrl}/realtime/v1/sessions/${encodeURIComponent(opts.addressingKey)}/in/append`, + { + method: "POST", + headers: { + Authorization: `Bearer ${opts.secretKey}`, + "Content-Type": "application/json", + "X-Part-Id": "close", + }, + body: JSON.stringify({ + kind: "message", + payload: { chatId: opts.addressingKey, trigger: "close" }, + }), + } + ); + } catch {} + runSignal.abort(); + await done.catch(() => {}); + }, + }; +} + +export type ChatAgentSessionOptions = Omit< + RunRealChatAgentOptions, + "runId" | "continuation" | "previousRunId" +>; + +export type ChatAgentSession = { + /** How many runs the session has spawned so far (1 fresh + N continuations). */ + runCount: () => number; + /** Close the currently-active run and stop spawning continuations. */ + close: () => Promise<void>; +}; + +/** + * A session-scoped orchestrator that stands in for the run-engine's run + * lifecycle: it starts a run, and whenever that run exits on its own + * (`chat.endRun()` / `chat.requestUpgrade()`), spawns the next run as a + * continuation (new run id, `continuation: true`, `previousRunId` threaded) + * for the same session. That mirrors the server triggering a fresh run on the + * next append after the previous run went terminal, and lets each continuation + * restore prior history from the persisted snapshot. Runs never overlap: the + * next spawn is chained on the previous run's `done` (after its manager + * teardown), so the process-global managers are never installed twice at once. + */ +export function runChatAgentSession(opts: ChatAgentSessionOptions): ChatAgentSession { + let closed = false; + let index = 0; + let current: RunningAgent | undefined; + let previousRunId: string | undefined; + + const spawn = () => { + index += 1; + const runId = `run_${opts.addressingKey}_${index}`; + current = runRealChatAgent({ + ...opts, + runId, + continuation: index > 1, + previousRunId, + }); + previousRunId = runId; + const settle = () => { + if (!closed) { + spawn(); + } + }; + current.done.then(settle, settle); + }; + + spawn(); + + return { + runCount: () => index, + close: async () => { + closed = true; + await current?.close(); + }, + }; +} diff --git a/apps/webapp/test/helpers/sessionStream.ts b/apps/webapp/test/helpers/sessionStream.ts new file mode 100644 index 00000000000..085c3cc4ed6 --- /dev/null +++ b/apps/webapp/test/helpers/sessionStream.ts @@ -0,0 +1,280 @@ +import { AppendInput, AppendRecord, S2 } from "@s2-dev/streamstore"; +import { generateJWT } from "@trigger.dev/core/v3/jwt"; +import { SSEStreamSubscription } from "@trigger.dev/core/v3"; + +export type SessionAddressing = { + orgId: string; + envSlug: string; + envId: string; + addressingKey: string; + io?: "out" | "in"; +}; + +/** + * The full, prefixed S2 stream name for a session channel on the shared basin + * (per-org basins disabled), matching `toSessionStreamName` + + * `streamPrefixFor` in the webapp. A test appending with the root S2 token uses + * this literal name. + */ +export function sessionStreamName(p: SessionAddressing): string { + return `org/${p.orgId}/env/${p.envSlug}/${p.envId}/sessions/${p.addressingKey}/${p.io ?? "out"}`; +} + +/** + * Mint a session-scoped public access token the way `mintSessionToken.server.ts` + * does: a JWT signed with the environment secret, `sub` = env id, `pub` true, + * scoped to read/write the given addressing key. + */ +export function mintSessionToken(p: { + apiKey: string; + envId: string; + addressingKey: string; +}): Promise<string> { + return generateJWT({ + secretKey: p.apiKey, + payload: { + pub: true, + sub: p.envId, + scopes: [`read:sessions:${p.addressingKey}`, `write:sessions:${p.addressingKey}`], + }, + expirationTime: "1h", + }); +} + +/** + * Writes `.out` records straight to S2 (the "agent simulator"). Uses the same + * `@s2-dev/streamstore` primitives the real agent runtime uses, so data, + * `trigger-control` and `trim` command records land in exactly the shapes the + * client + proxy expect. + */ +export class SessionStreamProducer { + private stream; + + constructor(p: { endpoint: string; basin: string; streamName: string; accessToken?: string }) { + const s2 = new S2({ + accessToken: p.accessToken ?? "ignored", + endpoints: { account: p.endpoint, basin: p.endpoint }, + }); + this.stream = s2.basin(p.basin).stream(p.streamName); + } + + /** Append one data record (`{data, id}` envelope). Returns its seq_num. */ + async appendData(data: unknown, id: string): Promise<number> { + const ack = await this.stream.append( + AppendInput.create([AppendRecord.string({ body: JSON.stringify({ data, id }) })]) + ); + return Number(ack.start.seqNum); + } + + /** Append a `trigger-control: turn-complete` record (empty body). */ + async appendTurnComplete(publicAccessToken?: string): Promise<number> { + const headers: Array<[string, string]> = [["trigger-control", "turn-complete"]]; + if (publicAccessToken) headers.push(["public-access-token", publicAccessToken]); + const ack = await this.stream.append( + AppendInput.create([AppendRecord.string({ body: "", headers })]) + ); + return Number(ack.start.seqNum); + } + + /** Append an S2 `trim` command record, trimming below `earliestSeqNum`. */ + async trim(earliestSeqNum: number): Promise<void> { + await this.stream.append(AppendInput.create([AppendRecord.trim(earliestSeqNum)])); + } +} + +export type CollectedPart = { + id: string; + chunk: unknown; + headers?: ReadonlyArray<readonly [string, string]>; +}; + +export function isTurnComplete(part: CollectedPart): boolean { + return (part.headers ?? []).some(([k, v]) => k === "trigger-control" && v === "turn-complete"); +} + +export function isUpgradeRequired(part: CollectedPart): boolean { + return (part.headers ?? []).some(([k, v]) => k === "trigger-control" && v === "upgrade-required"); +} + +export type SubscribeOptions = { + baseUrl: string; + addressingKey: string; + token: string; + lastEventId?: string; + timeoutInSeconds?: number; + peekSettled?: boolean; + io?: "out" | "in"; +}; + +function sessionChannelUrl(baseUrl: string, addressingKey: string, io: "out" | "in"): string { + return `${baseUrl}/realtime/v1/sessions/${encodeURIComponent(addressingKey)}/${io}`; +} + +/** + * Append a record to a session's `.in` channel through the webapp route the + * browser uses (client -> agent). Returns the status, the reflected + * Access-Control-Allow-Origin (when an Origin was sent), and the parsed body. + */ +export async function appendInput(opts: { + baseUrl: string; + addressingKey: string; + token: string; + body: string; + partId?: string; + origin?: string; +}): Promise<{ status: number; acao: string | null; json: unknown }> { + const url = `${sessionChannelUrl(opts.baseUrl, opts.addressingKey, "in")}/append`; + const res = await fetch(url, { + method: "POST", + headers: { + Authorization: `Bearer ${opts.token}`, + "Content-Type": "application/json", + ...(opts.partId ? { "X-Part-Id": opts.partId } : {}), + ...(opts.origin ? { Origin: opts.origin } : {}), + }, + body: opts.body, + }); + let json: unknown; + try { + json = await res.json(); + } catch {} + return { status: res.status, acao: res.headers.get("access-control-allow-origin"), json }; +} + +/** + * Raw SSE GET against a session channel, exposing the response status + + * headers (which `SSEStreamSubscription` hides). Reads the body to close and + * reports how long that took, for asserting the server-side peek fast-close. + */ +export async function openChannelRaw(opts: SubscribeOptions & { maxMs?: number }): Promise<{ + status: number; + sessionSettled: string | null; + closedMs: number; + body: string; + timedOut: boolean; +}> { + const url = sessionChannelUrl(opts.baseUrl, opts.addressingKey, opts.io ?? "out"); + const started = performance.now(); + const maxMs = opts.maxMs ?? 15_000; + const abort = new AbortController(); + const timer = setTimeout(() => abort.abort(), maxMs); + try { + const res = await fetch(url, { + signal: abort.signal, + headers: { + Authorization: `Bearer ${opts.token}`, + Accept: "text/event-stream", + ...(opts.lastEventId ? { "Last-Event-ID": opts.lastEventId } : {}), + ...(opts.timeoutInSeconds ? { "Timeout-Seconds": String(opts.timeoutInSeconds) } : {}), + ...(opts.peekSettled ? { "X-Peek-Settled": "1" } : {}), + }, + }); + const sessionSettled = res.headers.get("x-session-settled"); + let body = ""; + if (res.body) { + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + body += decoder.decode(value, { stream: true }); + } + } catch { + } finally { + await reader.cancel().catch(() => {}); + } + } + return { + status: res.status, + sessionSettled, + closedMs: performance.now() - started, + body, + timedOut: false, + }; + } catch { + return { + status: 0, + sessionSettled: null, + closedMs: performance.now() - started, + body: "", + timedOut: true, + }; + } finally { + clearTimeout(timer); + } +} + +export function subscribeSessionOut(opts: SubscribeOptions): SSEStreamSubscription { + const url = sessionChannelUrl(opts.baseUrl, opts.addressingKey, opts.io ?? "out"); + return new SSEStreamSubscription(url, { + headers: { + Authorization: `Bearer ${opts.token}`, + ...(opts.peekSettled ? { "X-Peek-Settled": "1" } : {}), + }, + timeoutInSeconds: opts.timeoutInSeconds ?? 30, + lastEventId: opts.lastEventId, + maxRetries: 0, + }); +} + +/** + * Subscribe + drain parts into an array, stopping when `until(parts)` is true + * or `maxMs` elapses. Cancels the reader on exit. + * + * The `.out` proxy serves a bounded `wait=N` window and then closes the SSE; + * real clients (and the run-engine session manager) reconnect with + * `Last-Event-ID` to keep draining a turn whose output streams across that + * boundary. A streaming agent routinely spans several windows, so when an + * `until` predicate is supplied this re-subscribes past each graceful close + * (deduping by seq, resuming from the last one seen) until the predicate holds + * or the deadline passes. Without `until` it reads a single window, matching a + * one-shot "read what's there" call. + */ +export async function collectSessionOut( + opts: SubscribeOptions & { until?: (parts: CollectedPart[]) => boolean; maxMs?: number } +): Promise<{ parts: CollectedPart[]; durationMs: number; subscription: SSEStreamSubscription }> { + const parts: CollectedPart[] = []; + const seen = new Set<string>(); + const started = performance.now(); + const deadline = started + (opts.maxMs ?? 30_000); + let lastEventId = opts.lastEventId; + let subscription: SSEStreamSubscription; + + do { + subscription = subscribeSessionOut({ ...opts, lastEventId }); + const stream = await subscription.subscribe(); + const reader = stream.getReader(); + let gotNew = false; + try { + while (true) { + if (opts.until && opts.until(parts)) break; + const remaining = deadline - performance.now(); + if (remaining <= 0) break; + const next = await Promise.race([ + reader.read(), + new Promise<"timeout">((r) => setTimeout(() => r("timeout"), remaining)), + ]); + if (next === "timeout") break; + if (next.done) break; + const part = next.value as CollectedPart; + if (part.id) { + if (seen.has(part.id)) continue; + seen.add(part.id); + lastEventId = part.id; + } + parts.push(part); + gotNew = true; + } + } finally { + await reader.cancel().catch(() => {}); + } + + if (!opts.until || opts.until(parts) || performance.now() >= deadline) break; + if (!gotNew) { + await new Promise((r) => setTimeout(r, 100)); + } + } while (true); + + return { parts, durationMs: performance.now() - started, subscription }; +} diff --git a/apps/webapp/test/helpers/testChatAgent.ts b/apps/webapp/test/helpers/testChatAgent.ts new file mode 100644 index 00000000000..8aebb713a96 --- /dev/null +++ b/apps/webapp/test/helpers/testChatAgent.ts @@ -0,0 +1,341 @@ +import { chat } from "@trigger.dev/sdk/ai"; +import { locals, OutOfMemoryError } from "@trigger.dev/core/v3"; +import { stepCountIs, streamText, tool, type LanguageModel, type UIMessage } from "ai"; +import { z } from "zod"; + +/** + * The model is injected through locals (an in-process, by-reference value) + * rather than clientData, because a `MockLanguageModelV3` can't survive the + * JSON round-trip through the real `.in/append` route. The harness sets this + * before the run starts. + */ +export const testChatModelLocal = locals.create<LanguageModel>("e2e-test-chat.model"); + +export type TestChatClientData = { hydrated?: UIMessage[] }; + +function firstText(m: UIMessage): string { + const p = m.parts?.[0]; + return p?.type === "text" ? p.text : ""; +} + +export const testChatAgent = chat + .withClientData({ + schema: z.custom<TestChatClientData>((v) => v == null || typeof v === "object"), + }) + .agent({ + id: "e2e-test-chat", + + onValidateMessages: async ({ messages }) => { + for (const m of messages) { + if (m.role === "user" && firstText(m).toLowerCase().includes("blocked-word")) { + throw new Error("Message blocked by content filter"); + } + } + return messages; + }, + + hydrateMessages: async ({ clientData, incomingMessages }) => { + if (!clientData?.hydrated) return incomingMessages; + const merged = [...clientData.hydrated]; + for (const m of incomingMessages) { + const idx = merged.findIndex((x) => x.id === m.id); + if (idx === -1) merged.push(m); + else merged[idx] = m; + } + return merged; + }, + + actionSchema: z.discriminatedUnion("type", [z.object({ type: z.literal("undo") })]), + + onAction: async ({ action }) => { + if (action.type === "undo") { + chat.history.slice(0, -2); + } + }, + + run: async ({ messages, signal }) => { + const model = locals.get(testChatModelLocal); + if (!model) { + throw new Error("test model not injected via locals"); + } + return streamText({ model, messages, abortSignal: signal }); + }, + }); + +/** + * Records suspend/resume lifecycle-hook fires (keyed by chatId so tests can + * filter to their own session). Appended to by {@link testSuspendHooksChatAgent}. + */ +export const suspendResumeEvents: Array<{ + chatId: string; + kind: "suspend" | "resume"; + phase: string; +}> = []; + +/** + * Short-idle agent that records `onChatSuspend` / `onChatResume` fires, so a + * test can assert the lifecycle hooks run around a real suspend/resume. + */ +export const testSuspendHooksChatAgent = chat.agent({ + id: "e2e-test-chat-suspend-hooks", + idleTimeoutInSeconds: 1, + preloadIdleTimeoutInSeconds: 1, + onChatSuspend: async ({ chatId, phase }) => { + suspendResumeEvents.push({ chatId, kind: "suspend", phase }); + }, + onChatResume: async ({ chatId, phase }) => { + suspendResumeEvents.push({ chatId, kind: "resume", phase }); + }, + run: async ({ messages, signal }) => { + const model = locals.get(testChatModelLocal); + if (!model) { + throw new Error("test model not injected via locals"); + } + return streamText({ model, messages, abortSignal: signal }); + }, +}); + +/** + * Throws `OutOfMemoryError` on the first attempt so the run fails the way the + * runtime detects for a machine swap, then succeeds on attempt 2 (the retry + * boots through the restore path because `ctx.attempt.number > 1`). + */ +export const testOomChatAgent = chat.agent({ + id: "e2e-test-chat-oom", + idleTimeoutInSeconds: 1, + preloadIdleTimeoutInSeconds: 1, + run: async ({ messages, signal, ctx }) => { + if (ctx.attempt.number === 1) { + throw new OutOfMemoryError(); + } + const model = locals.get(testChatModelLocal); + if (!model) { + throw new Error("test model not injected via locals"); + } + return streamText({ model, messages, abortSignal: signal }); + }, +}); + +/** + * Short idle window and a short `turnTimeout`, so a run with no incoming + * message suspends and then times out on the waitpoint, ending the run. + */ +export const testTimeoutChatAgent = chat.agent({ + id: "e2e-test-chat-timeout", + idleTimeoutInSeconds: 1, + preloadIdleTimeoutInSeconds: 1, + turnTimeout: "3s", + preloadTimeout: "3s", + run: async ({ messages, signal }) => { + const model = locals.get(testChatModelLocal); + if (!model) { + throw new Error("test model not injected via locals"); + } + return streamText({ model, messages, abortSignal: signal }); + }, +}); + +/** + * A minimal agent with no lifecycle hooks. With `hydrateMessages` absent, the + * default snapshot + `.out`/`.in` replay boot path is what restores prior + * history on a continuation run. + */ +export const testPlainChatAgent = chat.agent({ + id: "e2e-test-chat-plain", + run: async ({ messages, signal }) => { + const model = locals.get(testChatModelLocal); + if (!model) { + throw new Error("test model not injected via locals"); + } + return streamText({ model, messages, abortSignal: signal }); + }, +}); + +/** + * A plain agent with a 1-second idle window, so the turn loop falls through to + * the suspending `session.in.wait()` almost immediately instead of catching the + * next message in the warm once() window. Used to exercise the suspend/resume + * waitpoint path. + */ +export const testIdleChatAgent = chat.agent({ + id: "e2e-test-chat-idle", + idleTimeoutInSeconds: 1, + preloadIdleTimeoutInSeconds: 1, + run: async ({ messages, signal }) => { + const model = locals.get(testChatModelLocal); + if (!model) { + throw new Error("test model not injected via locals"); + } + return streamText({ model, messages, abortSignal: signal }); + }, +}); + +/** + * Ends the run after every turn via `chat.endRun()`. The next message on the + * same chat starts a fresh continuation run (the orchestrator drives that). + */ +export const testEndRunChatAgent = chat.agent({ + id: "e2e-test-chat-endrun", + idleTimeoutInSeconds: 2, + preloadIdleTimeoutInSeconds: 2, + run: async ({ messages, signal }) => { + const model = locals.get(testChatModelLocal); + if (!model) { + throw new Error("test model not injected via locals"); + } + const result = streamText({ model, messages, abortSignal: signal }); + chat.endRun(); + return result; + }, +}); + +/** + * Requests an upgrade from `onTurnStart` (the pre-turn path): `run()` is + * skipped, an `upgrade-required` control record lands on `.out`, and the run + * exits so a fresh run on the new version handles the message. + */ +export const testUpgradeChatAgent = chat.agent({ + id: "e2e-test-chat-upgrade", + idleTimeoutInSeconds: 2, + preloadIdleTimeoutInSeconds: 2, + onTurnStart: async () => { + chat.requestUpgrade(); + }, + run: async ({ messages, signal }) => { + const model = locals.get(testChatModelLocal); + if (!model) { + throw new Error("test model not injected via locals"); + } + return streamText({ model, messages, abortSignal: signal }); + }, +}); + +/** + * Requests an upgrade only on the fresh (non-continuation) run. The first run + * defers the message via `upgrade-required`; the continuation run treats it as + * the "new version" and processes the deferred message instead of upgrading + * again (which would loop). + */ +export const testUpgradeOnceChatAgent = chat.agent({ + id: "e2e-test-chat-upgrade-once", + idleTimeoutInSeconds: 2, + preloadIdleTimeoutInSeconds: 2, + onTurnStart: async ({ continuation }) => { + if (!continuation) { + chat.requestUpgrade(); + } + }, + run: async ({ messages, signal }) => { + const model = locals.get(testChatModelLocal); + if (!model) { + throw new Error("test model not injected via locals"); + } + return streamText({ model, messages, abortSignal: signal }); + }, +}); + +/** + * A tool with a server-side `execute`: the agent runs it automatically and + * feeds the result back to the model, so a single turn covers the whole + * tool loop. + */ +const weatherTool = tool({ + description: "Get the current weather for a city.", + inputSchema: z.object({ city: z.string() }), + execute: async ({ city }) => ({ city, tempC: 21, summary: "clear" }), +}); + +export const testToolChatAgent = chat.agent({ + id: "e2e-test-chat-tool", + run: async ({ messages, signal }) => { + const model = locals.get(testChatModelLocal); + if (!model) { + throw new Error("test model not injected via locals"); + } + return streamText({ + model, + messages, + tools: { getWeather: weatherTool }, + stopWhen: stepCountIs(5), + abortSignal: signal, + }); + }, +}); + +/** + * A tool with no `execute`: the model's call parks the turn on a + * human-in-the-loop round-trip. The client supplies the tool output on the + * next message and the agent continues from there. + */ +const askUserTool = tool({ + description: "Ask the user a question and wait for their answer.", + inputSchema: z.object({ question: z.string() }), +}); + +export const testHitlChatAgent = chat.agent({ + id: "e2e-test-chat-hitl", + run: async ({ messages, signal }) => { + const model = locals.get(testChatModelLocal); + if (!model) { + throw new Error("test model not injected via locals"); + } + return streamText({ + model, + messages, + tools: { askUser: askUserTool }, + abortSignal: signal, + }); + }, +}); + +/** + * Same HITL tool but with a 1-second idle window, so the run suspends on the + * waitpoint while waiting for the human's tool answer. Exercises a HITL + * round-trip that crosses a suspend/resume boundary. + */ +export const testHitlIdleChatAgent = chat.agent({ + id: "e2e-test-chat-hitl-idle", + idleTimeoutInSeconds: 1, + preloadIdleTimeoutInSeconds: 1, + run: async ({ messages, signal }) => { + const model = locals.get(testChatModelLocal); + if (!model) { + throw new Error("test model not injected via locals"); + } + return streamText({ + model, + messages, + tools: { askUser: askUserTool }, + abortSignal: signal, + }); + }, +}); + +/** + * A tool that both executes and requires approval. The model's call parks on + * an approval request; the client approves (or denies) before the `execute` + * runs. + */ +const deleteResourceTool = tool({ + description: "Delete a resource. Requires human approval before running.", + inputSchema: z.object({ resource: z.string() }), + needsApproval: true, + execute: async ({ resource }) => ({ deleted: resource }), +}); + +export const testApprovalChatAgent = chat.agent({ + id: "e2e-test-chat-approval", + run: async ({ messages, signal }) => { + const model = locals.get(testChatModelLocal); + if (!model) { + throw new Error("test model not injected via locals"); + } + return streamText({ + model, + messages, + tools: { deleteResource: deleteResourceTool }, + stopWhen: stepCountIs(5), + abortSignal: signal, + }); + }, +}); diff --git a/apps/webapp/test/session-agent.e2e.test.ts b/apps/webapp/test/session-agent.e2e.test.ts new file mode 100644 index 00000000000..f6a9a2a338c --- /dev/null +++ b/apps/webapp/test/session-agent.e2e.test.ts @@ -0,0 +1,1536 @@ +/** + * Tier-1 agent e2e: the real chat.agent turn loop against the full stack. + * + * Boots the testcontainer stack (webapp + Postgres + Redis + s2-lite + MinIO), + * runs the genuine `chat.agent` run loop in-process wired to the real webapp + * Session streams (real S2 `.out`/`.in` + object-store snapshots), with the + * language model injected as a deterministic MockLanguageModelV3. Turns are + * driven by appending to `.in` over HTTP; output is read back through the real + * SSE proxy. No real LLM, no `trigger dev`. + * + * Requires a pre-built webapp: pnpm run build --filter webapp + */ +import { randomBytes } from "crypto"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import "@trigger.dev/sdk/ai/test"; +import { MockLanguageModelV3 } from "ai/test"; +import { simulateReadableStream } from "ai"; +import type { SessionStreamTestServer } from "@internal/testcontainers/webapp"; +import { startSessionStreamTestServer } from "@internal/testcontainers/webapp"; +import { seedTestEnvironment } from "./helpers/seedTestEnvironment"; +import { + appendInput, + collectSessionOut, + isTurnComplete, + isUpgradeRequired, + mintSessionToken, + subscribeSessionOut, + type CollectedPart, +} from "./helpers/sessionStream"; +import { runChatAgentSession, runRealChatAgent } from "./helpers/agentHarness"; +import { + suspendResumeEvents, + testApprovalChatAgent, + testChatAgent, + testChatModelLocal, + testEndRunChatAgent, + testHitlChatAgent, + testHitlIdleChatAgent, + testIdleChatAgent, + testOomChatAgent, + testPlainChatAgent, + testSuspendHooksChatAgent, + testTimeoutChatAgent, + testToolChatAgent, + testUpgradeChatAgent, + testUpgradeOnceChatAgent, +} from "./helpers/testChatAgent"; + +async function waitFor(predicate: () => boolean, maxMs: number): Promise<void> { + const deadline = performance.now() + maxMs; + while (performance.now() < deadline) { + if (predicate()) return; + await new Promise((r) => setTimeout(r, 100)); + } +} + +vi.setConfig({ testTimeout: 120_000, hookTimeout: 240_000 }); + +let server: SessionStreamTestServer; + +beforeAll(async () => { + server = await startSessionStreamTestServer(); +}, 240_000); + +afterAll(async () => { + await server?.stop(); +}, 120_000); + +/** + * A synchronous mock model stream: enqueues every chunk in one tick and closes. + * `simulateReadableStream`'s default 0ms delay still yields a `setTimeout(0)` + * macrotask between chunks, which a loaded CI event loop can starve so the + * later chunks never emit and the turn produces an empty `.out`. Emitting + * synchronously keeps the mock deterministic across environments. + */ +function simStream(chunks: readonly unknown[]): ReadableStream<any> { + return new ReadableStream({ + start(controller) { + for (const chunk of chunks) { + controller.enqueue(chunk); + } + controller.close(); + }, + }); +} + +function textModel(text: string) { + const chunks = [ + { type: "text-start", id: "t1" }, + { type: "text-delta", id: "t1", delta: text }, + { type: "text-end", id: "t1" }, + { + type: "finish", + finishReason: { unified: "stop", raw: "stop" }, + usage: { + inputTokens: { total: 5, noCache: 5, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: 5, text: 5, reasoning: undefined }, + }, + }, + ]; + return new MockLanguageModelV3({ + doStream: async () => ({ stream: simStream(chunks) }), + }); +} + +async function setupSession(agentId: string = testChatAgent.id) { + const { organization, project, environment, apiKey } = await seedTestEnvironment(server.prisma); + const addressingKey = `chat-${randomBytes(6).toString("hex")}`; + await server.prisma.session.create({ + data: { + friendlyId: `session_${randomBytes(8).toString("hex")}`, + externalId: addressingKey, + type: "chat.agent", + projectId: project.id, + runtimeEnvironmentId: environment.id, + environmentType: environment.type, + organizationId: organization.id, + taskIdentifier: agentId, + triggerConfig: { basePayload: {} }, + }, + }); + const token = await mintSessionToken({ apiKey, envId: environment.id, addressingKey }); + return { addressingKey, token, apiKey, baseUrl: server.webapp.baseUrl }; +} + +function promptText(prompt: unknown): string { + if (!Array.isArray(prompt)) return ""; + let out = ""; + for (const m of prompt) { + const c = (m as { content?: unknown }).content; + if (typeof c === "string") out += `${c} `; + else if (Array.isArray(c)) { + for (const part of c) { + if (part && typeof part === "object" && (part as { type?: string }).type === "text") { + out += `${(part as { text?: string }).text ?? ""} `; + } + } + } + } + return out; +} + +function echoModel() { + return new MockLanguageModelV3({ + doStream: async ({ prompt }) => { + const text = promptText(prompt).trim(); + const chunks = [ + { type: "text-start", id: "t1" }, + { type: "text-delta", id: "t1", delta: `echo:${text}` }, + { type: "text-end", id: "t1" }, + { + type: "finish", + finishReason: { unified: "stop", raw: "stop" }, + usage: { + inputTokens: { total: 1, noCache: 1, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: 1, text: 1, reasoning: undefined }, + }, + }, + ]; + return { stream: simStream(chunks) }; + }, + }); +} + +function userMessage(text: string, id = "u0") { + return { id, role: "user", parts: [{ type: "text", text }] }; +} + +function submitBody(addressingKey: string, message: unknown, metadata: unknown = {}) { + return JSON.stringify({ + kind: "message", + payload: { message, chatId: addressingKey, trigger: "submit-message", metadata }, + }); +} + +function actionBody(addressingKey: string, action: unknown, metadata: unknown = {}) { + return JSON.stringify({ + kind: "message", + payload: { chatId: addressingKey, trigger: "action", action, metadata }, + }); +} + +const stopBody = JSON.stringify({ kind: "stop" }); + +/** + * Streams `words` as separate text-delta chunks with a delay between each, so + * a stop signal sent after the first chunk reliably truncates the turn before + * the model finishes. + */ +function slowModel(words: string[], delayMs: number) { + const chunks = [ + { type: "text-start", id: "t1" }, + ...words.map((delta) => ({ type: "text-delta", id: "t1", delta })), + { type: "text-end", id: "t1" }, + { + type: "finish", + finishReason: { unified: "stop", raw: "stop" }, + usage: { + inputTokens: { total: 5, noCache: 5, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: words.length, text: words.length, reasoning: undefined }, + }, + }, + ]; + return new MockLanguageModelV3({ + doStream: async () => ({ + stream: simulateReadableStream({ chunks: chunks as never, chunkDelayInMs: delayMs }), + }), + }); +} + +function chunkType(part: CollectedPart): string | undefined { + const c = part.chunk as { type?: unknown } | null; + return c && typeof c === "object" && typeof c.type === "string" ? c.type : undefined; +} + +const FINISH_STOP = { + type: "finish", + finishReason: { unified: "stop", raw: "stop" }, + usage: { + inputTokens: { total: 5, noCache: 5, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: 5, text: 5, reasoning: undefined }, + }, +}; + +/** + * A stateful model that emits a tool-call on its first `doStream` and a plain + * text response on every call after that. Drives both the automatic + * tool-execute loop (one turn) and the HITL round-trip (two turns). + */ +function toolCallThenText(opts: { + toolName: string; + toolCallId: string; + input: Record<string, unknown>; + finalText: string; +}) { + const inputJson = JSON.stringify(opts.input); + const call1 = [ + { type: "tool-input-start", id: opts.toolCallId, toolName: opts.toolName }, + { type: "tool-input-delta", id: opts.toolCallId, delta: inputJson }, + { type: "tool-input-end", id: opts.toolCallId }, + { type: "tool-call", toolCallId: opts.toolCallId, toolName: opts.toolName, input: inputJson }, + { + type: "finish", + finishReason: { unified: "tool-calls", raw: "tool_calls" }, + usage: { + inputTokens: { total: 5, noCache: 5, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: 5, text: 0, reasoning: undefined }, + }, + }, + ]; + const call2 = [ + { type: "text-start", id: "t1" }, + { type: "text-delta", id: "t1", delta: opts.finalText }, + { type: "text-end", id: "t1" }, + FINISH_STOP, + ]; + let idx = 0; + return new MockLanguageModelV3({ + doStream: async () => ({ + stream: simStream(idx++ === 0 ? call1 : call2), + }), + }); +} + +function joinChunks(parts: CollectedPart[]): string { + return parts + .filter((p) => p.chunk != null) + .map((p) => JSON.stringify(p.chunk)) + .join(""); +} + +/** A model that returns `texts[i]` on its i-th `doStream` call. */ +function sequenceModel(texts: string[]) { + let idx = 0; + return new MockLanguageModelV3({ + doStream: async () => { + const text = texts[Math.min(idx, texts.length - 1)]!; + idx++; + const chunks = [ + { type: "text-start", id: "t1" }, + { type: "text-delta", id: "t1", delta: text }, + { type: "text-end", id: "t1" }, + FINISH_STOP, + ]; + return { stream: simStream(chunks) }; + }, + }); +} + +function regenerateBody(addressingKey: string) { + return JSON.stringify({ + kind: "message", + payload: { chatId: addressingKey, trigger: "regenerate-message", metadata: {} }, + }); +} + +function findApprovalRequest( + parts: CollectedPart[] +): { approvalId: string; toolCallId: string } | undefined { + for (const p of parts) { + const c = p.chunk as { type?: string; approvalId?: string; toolCallId?: string } | null; + if (c && c.type === "tool-approval-request" && c.approvalId && c.toolCallId) { + return { approvalId: c.approvalId, toolCallId: c.toolCallId }; + } + } + return undefined; +} + +describe("session agent e2e (real chat.agent loop)", () => { + it("EA1: a real agent turn streams assistant text to .out", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(); + const agent = runRealChatAgent({ + agentId: testChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: textModel("hello from the agent"), + modelLocal: testChatModelLocal, + }); + + try { + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u0", + body: submitBody(addressingKey, userMessage("hi")), + }); + + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isTurnComplete), + maxMs: 30_000, + }); + + const text = parts + .filter((p) => p.chunk != null) + .map((p) => JSON.stringify(p.chunk)) + .join(""); + expect(text).toContain("hello from the agent"); + expect(parts.some(isTurnComplete)).toBe(true); + } finally { + await agent.close(); + } + }); + + it("EA2: two turns continue on the same run", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(); + const agent = runRealChatAgent({ + agentId: testChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: textModel("reply"), + modelLocal: testChatModelLocal, + }); + + try { + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("one", "u1")), + }); + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u2", + body: submitBody(addressingKey, userMessage("two", "u2")), + }); + + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.filter(isTurnComplete).length >= 2, + maxMs: 40_000, + }); + + expect(parts.filter(isTurnComplete)).toHaveLength(2); + const replyText = parts + .filter((p) => p.chunk != null) + .map((p) => JSON.stringify(p.chunk)) + .join(""); + expect((replyText.match(/reply/g) ?? []).length).toBeGreaterThanOrEqual(2); + const seqs = parts.map((p) => Number(p.id)); + expect(seqs).toEqual([...seqs].sort((a, b) => a - b)); + } finally { + await agent.close(); + } + }); + + it("EA3: hydrateMessages injects prior history into the turn", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(); + const agent = runRealChatAgent({ + agentId: testChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: echoModel(), + modelLocal: testChatModelLocal, + }); + + try { + const hydrated = [ + { id: "h1", role: "user", parts: [{ type: "text", text: "HISTORY-MARKER" }] }, + { id: "h2", role: "assistant", parts: [{ type: "text", text: "prior reply" }] }, + ]; + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("current question", "u1"), { hydrated }), + }); + + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isTurnComplete), + maxMs: 30_000, + }); + + const text = parts + .filter((p) => p.chunk != null) + .map((p) => JSON.stringify(p.chunk)) + .join(""); + expect(text).toContain("HISTORY-MARKER"); + expect(text).toContain("current question"); + } finally { + await agent.close(); + } + }); + + it("EA4: onValidateMessages rejection writes an error chunk, keeps the run alive", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(); + const agent = runRealChatAgent({ + agentId: testChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: textModel("should not appear"), + modelLocal: testChatModelLocal, + }); + + try { + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("this has a blocked-word in it", "u1")), + }); + + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isTurnComplete), + maxMs: 30_000, + }); + + const errorChunk = parts.find((p) => chunkType(p) === "error"); + expect(errorChunk, "an error chunk should be written to .out").toBeTruthy(); + expect(JSON.stringify((errorChunk as CollectedPart).chunk)).toContain("content filter"); + expect(parts.some(isTurnComplete), "the turn still completes after the error").toBe(true); + + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u2", + body: submitBody(addressingKey, userMessage("hello now", "u2")), + }); + const follow = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.filter(isTurnComplete).length >= 2, + maxMs: 30_000, + }); + expect(follow.parts.filter(isTurnComplete).length).toBeGreaterThanOrEqual(2); + } finally { + await agent.close(); + } + }); + + it("EA5: a custom action fires onAction and completes without a turn", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(); + const agent = runRealChatAgent({ + agentId: testChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: textModel("turn reply"), + modelLocal: testChatModelLocal, + }); + + try { + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("first", "u1")), + }); + await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isTurnComplete), + maxMs: 30_000, + }); + + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "act1", + body: actionBody(addressingKey, { type: "undo" }), + }); + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.filter(isTurnComplete).length >= 2, + maxMs: 30_000, + }); + + expect(parts.filter(isTurnComplete).length).toBeGreaterThanOrEqual(2); + expect( + parts.find((p) => chunkType(p) === "error"), + "no error on the action" + ).toBeFalsy(); + } finally { + await agent.close(); + } + }); + + it("EA6: a stop signal aborts the turn mid-stream but still completes it", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(); + const words = Array.from({ length: 12 }, (_, i) => ` w${i}`); + const agent = runRealChatAgent({ + agentId: testChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: slowModel(words, 250), + modelLocal: testChatModelLocal, + }); + + try { + const subscription = subscribeSessionOut({ baseUrl, addressingKey, token }); + const stream = await subscription.subscribe(); + const reader = stream.getReader(); + + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("stream something long", "u1")), + }); + + const parts: CollectedPart[] = []; + let sentStop = false; + const deadline = performance.now() + 40_000; + try { + while (performance.now() < deadline) { + const remaining = deadline - performance.now(); + const next = await Promise.race([ + reader.read(), + new Promise<"timeout">((r) => setTimeout(() => r("timeout"), remaining)), + ]); + if (next === "timeout" || next.done) break; + const part = next.value as CollectedPart; + parts.push(part); + if (!sentStop && chunkType(part) === "text-delta") { + sentStop = true; + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "stop1", + body: stopBody, + }); + } + if (parts.some(isTurnComplete)) break; + } + } finally { + await reader.cancel().catch(() => {}); + } + + expect(sentStop, "a text-delta arrived so a stop could be sent").toBe(true); + expect(parts.some(isTurnComplete), "the aborted turn still writes turn-complete").toBe(true); + const deltas = parts.filter((p) => chunkType(p) === "text-delta").length; + expect(deltas, "generation was cut short before all deltas streamed").toBeLessThan( + words.length + ); + } finally { + await agent.close(); + } + }); + + it("EA7: the agent runs a server-side tool and streams the final answer", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(testToolChatAgent.id); + const agent = runRealChatAgent({ + agentId: testToolChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: toolCallThenText({ + toolName: "getWeather", + toolCallId: "tc_weather_e2e", + input: { city: "Paris" }, + finalText: "It is 21C and clear in Paris", + }), + modelLocal: testChatModelLocal, + }); + + try { + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("what is the weather in Paris?", "u1")), + }); + + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isTurnComplete), + maxMs: 40_000, + }); + + const blob = joinChunks(parts); + expect(blob, "the tool call is streamed to .out").toContain("getWeather"); + expect(blob, "the tool result feeds a final answer").toContain( + "It is 21C and clear in Paris" + ); + expect(parts.some(isTurnComplete)).toBe(true); + } finally { + await agent.close(); + } + }); + + it("EA8: a HITL tool round-trip parks on the call and resumes from the client answer", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(testHitlChatAgent.id); + const toolCallId = "tc_ask_e2e"; + const agent = runRealChatAgent({ + agentId: testHitlChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: toolCallThenText({ + toolName: "askUser", + toolCallId, + input: { question: "what color?" }, + finalText: "blue it is", + }), + modelLocal: testChatModelLocal, + }); + + try { + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("pick a color", "u1")), + }); + const turn1 = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isTurnComplete), + maxMs: 30_000, + }); + const blob1 = joinChunks(turn1.parts); + expect(blob1, "turn 1 streams the tool call").toContain("askUser"); + expect(blob1).toContain(toolCallId); + + const answer = { + id: "a-answer", + role: "assistant", + parts: [ + { + type: "tool-askUser", + toolCallId, + state: "output-available", + input: { question: "what color?" }, + output: { color: "blue" }, + }, + ], + }; + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u2", + body: submitBody(addressingKey, answer), + }); + const turn2 = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.filter(isTurnComplete).length >= 2, + maxMs: 30_000, + }); + expect(joinChunks(turn2.parts), "turn 2 resumes with the final answer").toContain( + "blue it is" + ); + } finally { + await agent.close(); + } + }); + + it("EA9: a continuation run restores prior history from the persisted snapshot", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(testPlainChatAgent.id); + const runId1 = `run_persist_1_${addressingKey}`; + const runId2 = `run_persist_2_${addressingKey}`; + + const firstRun = runRealChatAgent({ + agentId: testPlainChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: textModel("assistant acknowledges"), + modelLocal: testChatModelLocal, + runId: runId1, + }); + try { + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("REMEMBER-THIS-42", "u1")), + }); + await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isTurnComplete), + maxMs: 30_000, + }); + } finally { + await firstRun.close(); + } + + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u2", + body: submitBody(addressingKey, userMessage("second question", "u2")), + }); + + const secondRun = runRealChatAgent({ + agentId: testPlainChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: echoModel(), + modelLocal: testChatModelLocal, + runId: runId2, + continuation: true, + previousRunId: runId1, + }); + try { + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.filter(isTurnComplete).length >= 2, + maxMs: 40_000, + }); + const blob = joinChunks(parts); + expect(blob, "the restored prior user message is back in the model prompt").toContain( + "REMEMBER-THIS-42" + ); + expect(blob, "the new turn also ran").toContain("second question"); + } finally { + await secondRun.close(); + } + }); + + it("EA10: regenerate re-runs the last user turn with a fresh model call", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(testPlainChatAgent.id); + const agent = runRealChatAgent({ + agentId: testPlainChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: sequenceModel(["first-answer", "regenerated-answer"]), + modelLocal: testChatModelLocal, + }); + + try { + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("explain it", "u1")), + }); + const first = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isTurnComplete), + maxMs: 30_000, + }); + expect(joinChunks(first.parts)).toContain("first-answer"); + + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "regen1", + body: regenerateBody(addressingKey), + }); + const second = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.filter(isTurnComplete).length >= 2, + maxMs: 30_000, + }); + expect(joinChunks(second.parts), "the regenerated turn produced a fresh answer").toContain( + "regenerated-answer" + ); + } finally { + await agent.close(); + } + }); + + it("EA11: a needsApproval tool parks on an approval request and runs once approved", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(testApprovalChatAgent.id); + const toolCallId = "tc_delete_e2e"; + const agent = runRealChatAgent({ + agentId: testApprovalChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: toolCallThenText({ + toolName: "deleteResource", + toolCallId, + input: { resource: "widget-1" }, + finalText: "deleted widget-1", + }), + modelLocal: testChatModelLocal, + }); + + try { + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("delete widget-1", "u1")), + }); + const turn1 = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isTurnComplete), + maxMs: 30_000, + }); + const approval = findApprovalRequest(turn1.parts); + expect(approval, "turn 1 emits an approval request instead of executing").toBeTruthy(); + expect(joinChunks(turn1.parts), "the tool did not execute before approval").not.toContain( + "deleted widget-1" + ); + + const answer = { + id: "a-approval", + role: "assistant", + parts: [ + { + type: "tool-deleteResource", + toolCallId: approval!.toolCallId, + state: "approval-responded", + approval: { id: approval!.approvalId, approved: true }, + }, + ], + }; + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u2", + body: submitBody(addressingKey, answer), + }); + const turn2 = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.filter(isTurnComplete).length >= 2, + maxMs: 30_000, + }); + expect( + joinChunks(turn2.parts), + "after approval the tool runs and the agent answers" + ).toContain("deleted widget-1"); + } finally { + await agent.close(); + } + }); + + it("EA12: two chats stay isolated - neither run bleeds into the other's .out", async () => { + const chatA = await setupSession(testPlainChatAgent.id); + const chatB = await setupSession(testPlainChatAgent.id); + + const agentA = runRealChatAgent({ + agentId: testPlainChatAgent.id, + baseUrl: chatA.baseUrl, + addressingKey: chatA.addressingKey, + secretKey: chatA.apiKey, + model: textModel("answer-for-A"), + modelLocal: testChatModelLocal, + }); + try { + await appendInput({ + baseUrl: chatA.baseUrl, + addressingKey: chatA.addressingKey, + token: chatA.token, + partId: "a1", + body: submitBody(chatA.addressingKey, userMessage("hi from A", "a1")), + }); + await collectSessionOut({ + baseUrl: chatA.baseUrl, + addressingKey: chatA.addressingKey, + token: chatA.token, + until: (p) => p.some(isTurnComplete), + maxMs: 30_000, + }); + } finally { + await agentA.close(); + } + + const agentB = runRealChatAgent({ + agentId: testPlainChatAgent.id, + baseUrl: chatB.baseUrl, + addressingKey: chatB.addressingKey, + secretKey: chatB.apiKey, + model: textModel("answer-for-B"), + modelLocal: testChatModelLocal, + }); + try { + await appendInput({ + baseUrl: chatB.baseUrl, + addressingKey: chatB.addressingKey, + token: chatB.token, + partId: "b1", + body: submitBody(chatB.addressingKey, userMessage("hi from B", "b1")), + }); + await collectSessionOut({ + baseUrl: chatB.baseUrl, + addressingKey: chatB.addressingKey, + token: chatB.token, + until: (p) => p.some(isTurnComplete), + maxMs: 30_000, + }); + } finally { + await agentB.close(); + } + + const outA = joinChunks( + ( + await collectSessionOut({ + baseUrl: chatA.baseUrl, + addressingKey: chatA.addressingKey, + token: chatA.token, + until: (p) => p.some(isTurnComplete), + maxMs: 15_000, + }) + ).parts + ); + const outB = joinChunks( + ( + await collectSessionOut({ + baseUrl: chatB.baseUrl, + addressingKey: chatB.addressingKey, + token: chatB.token, + until: (p) => p.some(isTurnComplete), + maxMs: 15_000, + }) + ).parts + ); + + expect(outA).toContain("answer-for-A"); + expect(outA, "A's stream never sees B's output").not.toContain("answer-for-B"); + expect(outB).toContain("answer-for-B"); + expect(outB, "B's stream never sees A's output").not.toContain("answer-for-A"); + }); + + it("EA13: the run suspends on the idle waitpoint, then the next message resumes it in place", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(testIdleChatAgent.id); + const agent = runRealChatAgent({ + agentId: testIdleChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: textModel("resumed after suspend"), + modelLocal: testChatModelLocal, + }); + + try { + await new Promise((r) => setTimeout(r, 2500)); + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("hi after idle", "u1")), + }); + + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isTurnComplete), + maxMs: 30_000, + }); + + expect( + joinChunks(parts), + "a message sent well after the idle window still produces a turn, so the waitpoint resume delivered it" + ).toContain("resumed after suspend"); + expect(parts.some(isTurnComplete)).toBe(true); + } finally { + await agent.close(); + } + }); + + it("EA14: chat.endRun() ends the run; the next message continues on a fresh run", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(testEndRunChatAgent.id); + const session = runChatAgentSession({ + agentId: testEndRunChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: echoModel(), + modelLocal: testChatModelLocal, + }); + + try { + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("MARKER-ONE", "u1")), + }); + await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isTurnComplete), + maxMs: 30_000, + }); + + await waitFor(() => session.runCount() >= 2, 10_000); + expect( + session.runCount(), + "endRun exited run 1, so the orchestrator spawned a continuation" + ).toBeGreaterThanOrEqual(2); + + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u2", + body: submitBody(addressingKey, userMessage("second msg", "u2")), + }); + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.filter(isTurnComplete).length >= 2, + maxMs: 30_000, + }); + + const blob = joinChunks(parts); + expect(blob, "the continuation run restored run 1's history").toContain("MARKER-ONE"); + expect(blob, "the continuation run ran the new turn").toContain("second msg"); + } finally { + await session.close(); + } + }); + + it("EA15: chat.requestUpgrade() emits upgrade-required on .out and exits the run", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(testUpgradeChatAgent.id); + const agent = runRealChatAgent({ + agentId: testUpgradeChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: echoModel(), + modelLocal: testChatModelLocal, + }); + + let exitedOnItsOwn = false; + agent.done.then(() => { + exitedOnItsOwn = true; + }); + + try { + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("upgrade me", "u1")), + }); + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isUpgradeRequired), + maxMs: 30_000, + }); + + expect( + parts.some(isUpgradeRequired), + "an upgrade-required control record is written to .out" + ).toBe(true); + + await waitFor(() => exitedOnItsOwn, 10_000); + expect(exitedOnItsOwn, "the run exits itself after requesting the upgrade").toBe(true); + } finally { + await agent.close(); + } + }); + + it("EA16: a HITL tool approval survives a suspend/resume boundary", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(testHitlIdleChatAgent.id); + const toolCallId = "tc_ask_suspend"; + const agent = runRealChatAgent({ + agentId: testHitlIdleChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: toolCallThenText({ + toolName: "askUser", + toolCallId, + input: { question: "what color?" }, + finalText: "blue it is", + }), + modelLocal: testChatModelLocal, + }); + + try { + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("pick a color", "u1")), + }); + const turn1 = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isTurnComplete), + maxMs: 30_000, + }); + expect(joinChunks(turn1.parts), "turn 1 parks on the tool call").toContain("askUser"); + + await new Promise((r) => setTimeout(r, 2500)); + + const answer = { + id: "a-answer", + role: "assistant", + parts: [ + { + type: "tool-askUser", + toolCallId, + state: "output-available", + input: { question: "what color?" }, + output: { color: "blue" }, + }, + ], + }; + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u2", + body: submitBody(addressingKey, answer), + }); + const turn2 = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => joinChunks(p).includes("blue it is"), + maxMs: 30_000, + }); + expect( + joinChunks(turn2.parts), + "the answer sent after the idle window resumed the suspended run" + ).toContain("blue it is"); + } finally { + await agent.close(); + } + }); + + it("EA17: the run suspends and resumes across multiple turns", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(testIdleChatAgent.id); + const replies = ["reply-one", "reply-two", "reply-three"]; + const agent = runRealChatAgent({ + agentId: testIdleChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: sequenceModel(replies), + modelLocal: testChatModelLocal, + }); + + try { + for (let i = 0; i < replies.length; i++) { + await new Promise((r) => setTimeout(r, i === 0 ? 1500 : 2000)); + await appendInput({ + baseUrl, + addressingKey, + token, + partId: `u${i + 1}`, + body: submitBody(addressingKey, userMessage(`turn ${i + 1}`, `u${i + 1}`)), + }); + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => joinChunks(p).includes(replies[i]!), + maxMs: 30_000, + }); + expect( + joinChunks(parts), + `turn ${i + 1} resumed from a suspend and produced its reply` + ).toContain(replies[i]!); + } + } finally { + await agent.close(); + } + }); + + it("EA18: requestUpgrade defers the message; the upgraded run processes it", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession( + testUpgradeOnceChatAgent.id + ); + const session = runChatAgentSession({ + agentId: testUpgradeOnceChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: echoModel(), + modelLocal: testChatModelLocal, + }); + + try { + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("DEFER-ME", "u1")), + }); + const turn1 = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isUpgradeRequired), + maxMs: 30_000, + }); + expect( + turn1.parts.some(isUpgradeRequired), + "the fresh run defers the message with upgrade-required" + ).toBe(true); + + await waitFor(() => session.runCount() >= 2, 10_000); + + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => joinChunks(p).includes("DEFER-ME"), + maxMs: 30_000, + }); + expect( + joinChunks(parts), + "the continuation run processed the deferred message instead of upgrading again" + ).toContain("DEFER-ME"); + } finally { + await session.close(); + } + }); + + it("EA19: endRun continuation restores history across multiple hops", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(testEndRunChatAgent.id); + const session = runChatAgentSession({ + agentId: testEndRunChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: echoModel(), + modelLocal: testChatModelLocal, + }); + + try { + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("HOP-MARKER", "u1")), + }); + await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isTurnComplete), + maxMs: 30_000, + }); + + await waitFor(() => session.runCount() >= 2, 10_000); + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u2", + body: submitBody(addressingKey, userMessage("second hop", "u2")), + }); + await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => joinChunks(p).includes("second hop"), + maxMs: 30_000, + }); + + await waitFor(() => session.runCount() >= 3, 10_000); + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u3", + body: submitBody(addressingKey, userMessage("third hop", "u3")), + }); + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => joinChunks(p).includes("third hop"), + maxMs: 30_000, + }); + + const blob = joinChunks(parts); + expect(blob, "history from the first hop is restored two continuations later").toContain( + "HOP-MARKER" + ); + expect(blob).toContain("third hop"); + expect(session.runCount()).toBeGreaterThanOrEqual(3); + } finally { + await session.close(); + } + }); + + it("EA20: onChatSuspend and onChatResume fire around a suspend/resume", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession( + testSuspendHooksChatAgent.id + ); + const agent = runRealChatAgent({ + agentId: testSuspendHooksChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: sequenceModel(["turn-a", "turn-b"]), + modelLocal: testChatModelLocal, + }); + + try { + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("first", "u1")), + }); + await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isTurnComplete), + maxMs: 30_000, + }); + + await new Promise((r) => setTimeout(r, 2500)); + + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u2", + body: submitBody(addressingKey, userMessage("second", "u2")), + }); + await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => joinChunks(p).includes("turn-b"), + maxMs: 30_000, + }); + + const mine = suspendResumeEvents.filter((e) => e.chatId === addressingKey); + expect( + mine.some((e) => e.kind === "suspend"), + "onChatSuspend fired when the run suspended" + ).toBe(true); + expect( + mine.some((e) => e.kind === "resume"), + "onChatResume fired when the next message resumed it" + ).toBe(true); + } finally { + await agent.close(); + } + }); + + it("EA21: a preloaded run's OOM retry recovers the in-flight message", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(testOomChatAgent.id); + const runId = `run_oom_${addressingKey}`; + + const attempt1 = runRealChatAgent({ + agentId: testOomChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: echoModel(), + modelLocal: testChatModelLocal, + runId, + attemptNumber: 1, + }); + let attempt1Error: unknown; + attempt1.done.catch((e) => { + attempt1Error = e; + }); + + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("OOM-THEN-OK", "u1")), + }); + await waitFor(() => attempt1Error !== undefined, 15_000); + expect( + String(attempt1Error), + "attempt 1 fails with an OOM so the runtime can swap machines" + ).toMatch(/OutOfMemory/i); + + const attempt2 = runRealChatAgent({ + agentId: testOomChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: echoModel(), + modelLocal: testChatModelLocal, + runId, + attemptNumber: 2, + }); + try { + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => joinChunks(p).includes("OOM-THEN-OK"), + maxMs: 30_000, + }); + expect( + joinChunks(parts), + "the attempt-2 retry restored the unprocessed message and ran it" + ).toContain("OOM-THEN-OK"); + } finally { + await attempt2.close(); + } + }); + + it("EA22: the idle wait times out and ends the run when no message arrives", async () => { + const { addressingKey, token, apiKey, baseUrl } = await setupSession(testTimeoutChatAgent.id); + const agent = runRealChatAgent({ + agentId: testTimeoutChatAgent.id, + baseUrl, + addressingKey, + secretKey: apiKey, + model: textModel("turn one"), + modelLocal: testChatModelLocal, + }); + + let ended = false; + agent.done.then(() => { + ended = true; + }); + + try { + await appendInput({ + baseUrl, + addressingKey, + token, + partId: "u1", + body: submitBody(addressingKey, userMessage("hello", "u1")), + }); + await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isTurnComplete), + maxMs: 30_000, + }); + + await waitFor(() => ended, 20_000); + expect( + ended, + "with no next message the between-turns wait times out and the run exits itself" + ).toBe(true); + } finally { + await agent.close(); + } + }); +}); diff --git a/apps/webapp/test/session-stream.browser.e2e.test.ts b/apps/webapp/test/session-stream.browser.e2e.test.ts new file mode 100644 index 00000000000..4eeebf826ae --- /dev/null +++ b/apps/webapp/test/session-stream.browser.e2e.test.ts @@ -0,0 +1,133 @@ +/** + * Browser-level session-stream e2e. + * + * Same full stack as session-stream.e2e.test.ts, but the client is a real + * Chromium page on a DIFFERENT origin than the webapp, driven via + * page.evaluate. It exercises what node-fetch can't: a real browser doing a + * cross-origin fetch + streaming read of the session `.out` SSE through the + * webapp proxy, so the browser enforces the CORS preflight + response headers + * the customer scenario depends on (a frontend on its own origin subscribing + * to the API). The webapp runs production-mode here, so this checks the + * production CORS path, not the dev-server one. + * + * Requires a pre-built webapp (pnpm run build --filter webapp) and Chromium + * (pnpm exec playwright install chromium). + */ +import { randomBytes } from "crypto"; +import { createServer, type Server } from "http"; +import { chromium, type Browser } from "@playwright/test"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import type { SessionStreamTestServer } from "@internal/testcontainers/webapp"; +import { startSessionStreamTestServer } from "@internal/testcontainers/webapp"; +import { seedTestEnvironment } from "./helpers/seedTestEnvironment"; +import { + mintSessionToken, + SessionStreamProducer, + sessionStreamName, +} from "./helpers/sessionStream"; + +vi.setConfig({ testTimeout: 120_000, hookTimeout: 240_000 }); + +let server: SessionStreamTestServer; +let browser: Browser | undefined; +let origin: Server; +let originUrl: string; + +beforeAll(async () => { + server = await startSessionStreamTestServer(); + try { + browser = await chromium.launch(); + } catch (error) { + console.warn("[browser-e2e] Chromium unavailable, leg will skip:", String(error)); + } + origin = createServer((_req, res) => { + res.writeHead(200, { "content-type": "text/html" }); + res.end("<!doctype html><html><body>origin</body></html>"); + }); + await new Promise<void>((resolve) => origin.listen(0, "127.0.0.1", () => resolve())); + const addr = origin.address(); + const port = typeof addr === "object" && addr ? addr.port : 0; + originUrl = `http://127.0.0.1:${port}`; +}, 240_000); + +afterAll(async () => { + await browser?.close().catch(() => {}); + await new Promise<void>((resolve) => (origin ? origin.close(() => resolve()) : resolve())); + await server?.stop(); +}, 120_000); + +describe("session stream browser e2e", () => { + it("EB1: a cross-origin browser subscribes to .out and streams records", async (ctx) => { + if (!browser) { + ctx.skip(); + return; + } + const { organization, environment, apiKey } = await seedTestEnvironment(server.prisma); + const addressingKey = `sess-${randomBytes(6).toString("hex")}`; + const token = await mintSessionToken({ apiKey, envId: environment.id, addressingKey }); + const streamName = sessionStreamName({ + orgId: organization.id, + envSlug: environment.slug, + envId: environment.id, + addressingKey, + }); + const producer = new SessionStreamProducer({ + endpoint: server.s2.endpoint, + basin: server.s2.basin, + streamName, + }); + + await producer.appendData({ n: 0 }, "p0"); + await producer.appendData({ n: 1 }, "p1"); + await producer.appendTurnComplete(); + + const sseUrl = `${server.webapp.baseUrl}/realtime/v1/sessions/${encodeURIComponent( + addressingKey + )}/out`; + + const page = await browser.newPage(); + try { + await page.goto(originUrl); + const result = await page.evaluate( + async ({ url, token }) => { + const ac = new AbortController(); + const deadlineTimer = setTimeout(() => ac.abort(), 8000); + try { + const res = await fetch(url, { + headers: { Authorization: `Bearer ${token}`, Accept: "text/event-stream" }, + signal: ac.signal, + }); + const reader = (res.body as ReadableStream<Uint8Array>).getReader(); + const decoder = new TextDecoder(); + let text = ""; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + if (text.includes('"records"')) break; + } + ac.abort(); + return { + ok: res.ok, + status: res.status, + sawBatch: text.includes('"records"'), + pageOrigin: location.origin, + }; + } catch (e) { + return { error: String(e) }; + } finally { + clearTimeout(deadlineTimer); + } + }, + { url: sseUrl, token } + ); + + expect("error" in result ? result.error : undefined).toBeUndefined(); + expect(result).toMatchObject({ ok: true, status: 200, sawBatch: true }); + expect((result as { pageOrigin: string }).pageOrigin).toBe(originUrl); + expect(originUrl).not.toBe(server.webapp.baseUrl); + } finally { + await page.close(); + } + }); +}); diff --git a/apps/webapp/test/session-stream.e2e.test.ts b/apps/webapp/test/session-stream.e2e.test.ts new file mode 100644 index 00000000000..af1b631ee1e --- /dev/null +++ b/apps/webapp/test/session-stream.e2e.test.ts @@ -0,0 +1,326 @@ +/** + * Full-stack session-stream e2e. + * + * Boots the real webapp + Postgres + Redis + s2-lite 0.40.0 (via + * startSessionStreamTestServer), then drives the session `.out` wire protocol + * directly: a producer appends records straight to S2 (the agent simulator), + * and the client subscribes through the webapp SSE proxy using the real + * `SSEStreamSubscription` from `@trigger.dev/core` — the same code the browser + * runs, minus the DOM. + * + * Requires a pre-built webapp: pnpm run build --filter webapp + */ +import { randomBytes } from "crypto"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { SessionStreamInstance } from "@trigger.dev/core/v3"; +import type { SessionStreamTestServer } from "@internal/testcontainers/webapp"; +import { startSessionStreamTestServer } from "@internal/testcontainers/webapp"; +import { seedTestEnvironment } from "./helpers/seedTestEnvironment"; +import { + appendInput, + collectSessionOut, + isTurnComplete, + mintSessionToken, + openChannelRaw, + SessionStreamProducer, + sessionStreamName, +} from "./helpers/sessionStream"; + +vi.setConfig({ testTimeout: 120_000, hookTimeout: 180_000 }); + +let server: SessionStreamTestServer; + +beforeAll(async () => { + server = await startSessionStreamTestServer(); +}, 180_000); + +afterAll(async () => { + await server?.stop(); +}, 120_000); + +async function setupSession() { + const { organization, project, environment, apiKey } = await seedTestEnvironment(server.prisma); + const addressingKey = `sess-${randomBytes(6).toString("hex")}`; + await server.prisma.session.create({ + data: { + friendlyId: `session_${randomBytes(8).toString("hex")}`, + externalId: addressingKey, + type: "chat.agent", + projectId: project.id, + runtimeEnvironmentId: environment.id, + environmentType: environment.type, + organizationId: organization.id, + taskIdentifier: "chat-agent", + triggerConfig: { basePayload: {} }, + }, + }); + const token = await mintSessionToken({ apiKey, envId: environment.id, addressingKey }); + const streamName = sessionStreamName({ + orgId: organization.id, + envSlug: environment.slug, + envId: environment.id, + addressingKey, + }); + const producer = new SessionStreamProducer({ + endpoint: server.s2.endpoint, + basin: server.s2.basin, + streamName, + }); + return { addressingKey, token, producer, streamName, baseUrl: server.webapp.baseUrl }; +} + +function readableFrom<T>(chunks: T[]): ReadableStream<T> { + return new ReadableStream<T>({ + start(controller) { + for (const chunk of chunks) controller.enqueue(chunk); + controller.close(); + }, + }); +} + +describe("session stream e2e", () => { + it("E1 basic: data records + turn-complete are delivered in order", async () => { + const { addressingKey, token, producer, baseUrl } = await setupSession(); + + await producer.appendData({ n: 0 }, "p0"); + await producer.appendData({ n: 1 }, "p1"); + await producer.appendData({ n: 2 }, "p2"); + await producer.appendTurnComplete(); + + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isTurnComplete), + maxMs: 20_000, + }); + + const dataChunks = parts + .filter((p) => !isTurnComplete(p) && p.chunk != null) + .map((p) => (p.chunk as { n: number }).n); + + expect(dataChunks).toEqual([0, 1, 2]); + expect(parts.some(isTurnComplete)).toBe(true); + + const seqs = parts.map((p) => Number(p.id)); + expect(seqs).toEqual([...seqs].sort((a, b) => a - b)); + }); + + it("E2 continuation: seq is monotonic across two turns on the same .out", async () => { + const { addressingKey, token, producer, baseUrl } = await setupSession(); + + await producer.appendData({ n: 0 }, "p0"); + await producer.appendTurnComplete(); + await producer.appendData({ n: 1 }, "p1"); + await producer.appendData({ n: 2 }, "p2"); + const tc2 = await producer.appendTurnComplete(); + + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.filter(isTurnComplete).length >= 2, + maxMs: 20_000, + }); + + expect(parts.filter(isTurnComplete)).toHaveLength(2); + const seqs = parts.map((p) => Number(p.id)); + expect(seqs).toEqual([...seqs].sort((a, b) => a - b)); + expect(Math.max(...seqs)).toBe(tc2); + const dataChunks = parts + .filter((p) => !isTurnComplete(p) && p.chunk != null) + .map((p) => (p.chunk as { n: number }).n); + expect(dataChunks).toEqual([0, 1, 2]); + }); + + it("E5 no-dup reconnect: resume via Last-Event-ID delivers each record once", async () => { + const { addressingKey, token, producer, baseUrl } = await setupSession(); + + await producer.appendData({ n: 0 }, "p0"); + await producer.appendData({ n: 1 }, "p1"); + const s2 = await producer.appendData({ n: 2 }, "p2"); + + const first = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.filter((x) => x.chunk != null).length >= 3, + maxMs: 20_000, + }); + const firstSeqs = first.parts.map((p) => Number(p.id)); + expect(firstSeqs).toContain(s2); + + await producer.appendData({ n: 3 }, "p3"); + await producer.appendData({ n: 4 }, "p4"); + + const second = await collectSessionOut({ + baseUrl, + addressingKey, + token, + lastEventId: String(s2), + until: (p) => p.filter((x) => x.chunk != null).length >= 2, + maxMs: 20_000, + }); + const secondSeqs = second.parts.map((p) => Number(p.id)); + + const all = [...firstSeqs, ...secondSeqs]; + expect(new Set(all).size).toBe(all.length); + const allData = [...first.parts, ...second.parts] + .filter((p) => p.chunk != null) + .map((p) => (p.chunk as { n: number }).n) + .sort((a, b) => a - b); + expect(allData).toEqual([0, 1, 2, 3, 4]); + }); + + it("E7 trim-at-tail: trim command record is filtered from the stream", async () => { + const { addressingKey, token, producer, baseUrl } = await setupSession(); + + await producer.appendData({ n: 0 }, "p0"); + const keep = await producer.appendData({ n: 1 }, "p1"); + await producer.appendTurnComplete(); + await producer.trim(keep); + + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + until: (p) => p.some(isTurnComplete), + maxMs: 15_000, + }); + + const commandRecords = parts.filter((p) => (p.headers ?? []).some(([k]) => k === "")); + expect(commandRecords).toHaveLength(0); + expect(parts.filter((p) => p.chunk != null).length).toBeGreaterThanOrEqual(1); + }); + + it("E8 in/append 413 for an oversized body still carries CORS headers", async () => { + const { addressingKey, token, baseUrl } = await setupSession(); + + const oversized = "x".repeat(2 * 1024 * 1024); + const { status, acao } = await appendInput({ + baseUrl, + addressingKey, + token, + origin: "http://example.com", + body: JSON.stringify({ kind: "message", payload: { big: oversized } }), + }); + + expect(status).toBe(413); + expect(acao).not.toBeNull(); + }); + + it("E9 server peek fast-closes at a turn-complete tail with X-Session-Settled", async () => { + const { addressingKey, token, producer, baseUrl } = await setupSession(); + + await producer.appendData({ n: 0 }, "p0"); + const tc = await producer.appendTurnComplete(); + + let result: Awaited<ReturnType<typeof openChannelRaw>> | undefined; + for (let attempt = 0; attempt < 3; attempt++) { + result = await openChannelRaw({ + baseUrl, + addressingKey, + token, + lastEventId: String(tc), + peekSettled: true, + timeoutInSeconds: 30, + maxMs: 8_000, + }); + if (result.status === 200 && result.sessionSettled === "true") break; + await new Promise((r) => setTimeout(r, 500)); + } + + expect(result!.status).toBe(200); + expect(result!.sessionSettled).toBe("true"); + expect(result!.closedMs).toBeLessThan(8_000); + }); + + /** + * E10 head-start handover: the resume cursor the S2 stream writer reports + * from `wait()` must point AT the last record it wrote, not one past it. + * `chat.ln`'s warm process drains step 1 to `session.out`, hands that + * cursor to the agent's resume subscribe, and the read proxy resumes from + * `cursor + 1`. S2's append `end` is exclusive (last seq + 1), so if the + * writer reports `end` the agent's first post-handover record is skipped. + */ + it("E10 head-start handover cursor does not skip the first post-handover record", async () => { + const { addressingKey, token, producer, streamName, baseUrl } = await setupSession(); + + const writer = new SessionStreamInstance<{ n: number }>({ + apiClient: undefined as never, + baseUrl, + sessionId: addressingKey, + io: "out", + source: readableFrom([{ n: 0 }, { n: 1 }]), + initializeSession: async () => ({ + headers: { + "x-s2-access-token": "ignored", + "x-s2-basin": server.s2.basin, + "x-s2-stream-name": streamName, + "x-s2-endpoint": server.s2.endpoint, + }, + }), + }); + + const { lastEventId } = await writer.wait(); + expect(lastEventId).toBeDefined(); + + const agentSeq = await producer.appendData({ n: 2 }, "agent-0"); + + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + lastEventId, + until: (p) => p.some((x) => x.chunk != null), + maxMs: 15_000, + }); + + const dataChunks = parts + .filter((p) => p.chunk != null) + .map((p) => (p.chunk as { n: number }).n); + + expect(dataChunks).toEqual([2]); + expect(parts.map((p) => Number(p.id))).toContain(agentSeq); + }); + + it("E11 in/append delivers the record on the .in channel", async () => { + const { addressingKey, token, baseUrl } = await setupSession(); + + const payload = JSON.stringify({ kind: "message", text: "hello from client" }); + const appended = await appendInput({ + baseUrl, + addressingKey, + token, + partId: "in-0", + body: payload, + }); + expect(appended.status).toBe(200); + + const { parts } = await collectSessionOut({ + baseUrl, + addressingKey, + token, + io: "in", + until: (p) => p.some((x) => x.chunk != null), + maxMs: 15_000, + }); + + const got = parts.find((p) => p.chunk != null); + expect(got).toBeTruthy(); + expect(String(got?.chunk)).toContain("hello from client"); + }); + + it("E12 subscribe with an invalid token is rejected", async () => { + const { addressingKey, baseUrl } = await setupSession(); + + const { status } = await openChannelRaw({ + baseUrl, + addressingKey, + token: "tr_pub_invalid_not_a_real_token", + maxMs: 5_000, + }); + + expect([401, 403]).toContain(status); + }); +}); diff --git a/apps/webapp/vite.config.ts b/apps/webapp/vite.config.ts index b008ae04ed0..0b33a4edb6f 100644 --- a/apps/webapp/vite.config.ts +++ b/apps/webapp/vite.config.ts @@ -30,6 +30,7 @@ export default defineConfig({ }, }, server: { + cors: false, warmup: { clientFiles: ["./app/entry.client.tsx", "./app/root.tsx", "./app/components/**/*.tsx"], ssrFiles: ["./app/entry.server.tsx", "./app/root.tsx"], diff --git a/apps/webapp/vitest.e2e.config.ts b/apps/webapp/vitest.e2e.config.ts index 0c5c1e86c84..905c4ac6f22 100644 --- a/apps/webapp/vitest.e2e.config.ts +++ b/apps/webapp/vitest.e2e.config.ts @@ -6,6 +6,13 @@ export default defineConfig({ include: ["test/**/*.e2e.test.ts"], globals: true, pool: "forks", + /** + * Each e2e file boots its own Docker stack (Postgres, Redis, s2-lite, + * MinIO) plus a webapp process. Running files in parallel boots several + * stacks at once and thrashes the CI runner into a timeout, so run them + * one at a time, matching the repo's `pnpm test --no-file-parallelism`. + */ + fileParallelism: false, }, // @ts-ignore plugins: [tsconfigPaths({ projects: ["./tsconfig.json"] })], diff --git a/docs/ai-chat/client-protocol.mdx b/docs/ai-chat/client-protocol.mdx index fd8326e4ba9..d039b39366a 100644 --- a/docs/ai-chat/client-protocol.mdx +++ b/docs/ai-chat/client-protocol.mdx @@ -320,7 +320,7 @@ The output stream uses [S2](https://s2.dev) under the hood and follows the stand | Event | Meaning | | --- | --- | | `batch` | One or more records. The records you actually care about. | -| `ping` | Keepalive (~every 5s on idle). Body is `{"timestamp": <ms>}`. Ignore it. | +| `ping` | Keepalive (~every 5s on idle). Body is `{"timestamp": <ms>}`, and on backends that report it a `tail` (same `{seq_num, timestamp}` shape as a batch tail). The tail is what lets you tell you have caught up to the live edge; if you don't need caught-up detection, ignore the ping. | | _(no `event:`, just `data: [DONE]`)_ | Stream is closing — server sends this once before EOF. | A `batch` event in raw SSE format looks like this — note the `data` is a single line of JSON, no embedded newlines (per the SSE spec): @@ -359,7 +359,7 @@ Decoded `data` payload: | `records[].timestamp` | Unix ms when the record was written to S2. | | `records[].body` | For data records: a JSON-encoded **string** wrapping `{ data: UIMessageChunk, id: string }`. For control records: an empty string (semantics live in `headers`). For S2 command records: opaque bytes. See [Records on session.out](#records-on-session-out). | | `records[].headers` | Optional `[name, value]` pairs. Empty for data records; a `trigger-control` entry for control records; a single empty-name `["", "<op>"]` entry for S2 command records. | -| `tail.seq_num` | Latest known tail of the S2 stream — useful for detecting how far behind the live edge you are. Skip if you don't need it. | +| `tail.seq_num` | Latest known tail of the S2 stream, useful for detecting how far behind the live edge you are. Track the **highest `seq_num` you have received**, counting every record in the batch (including the command records you skip, see below), not just the last application-visible one. When `highest received seq_num + 1 === tail.seq_num` you have drained the backlog and are caught up to the live edge. The same `tail` also rides on `ping` events. Skip if you don't need it. | | `tail.timestamp` | Timestamp of `tail.seq_num`. | ### Records on `session.out` @@ -652,6 +652,8 @@ On **reconnect-on-reload** paths (resuming a chat where nothing may be streaming **Do not send `X-Peek-Settled` on the active-send response-stream path.** The peek would race the newly-triggered turn's first chunk — if the agent hasn't written the new turn's first record yet, the peek sees the prior turn's `turn-complete` and closes the SSE before the response lands on S2. The built-in `TriggerChatTransport.reconnectToStream` sets the header; `sendMessages → subscribeToStream` does not. +If you use `TriggerChatTransport` (or `useChat`, which builds on it), this is handled for you: the reconnect path sends `X-Peek-Settled`, so a settled idle reconnect closes promptly on the server's `wait=0` fast-close instead of waiting out the long poll. Consuming `.out` directly, send the same header on your reconnect reads and treat an `X-Session-Settled: true` response as a terminal close. + ```ts // Reconnect path (page reload) const response = await fetch(sseUrl, { diff --git a/internal-packages/testcontainers/src/s2.ts b/internal-packages/testcontainers/src/s2.ts new file mode 100644 index 00000000000..fe2a0494774 --- /dev/null +++ b/internal-packages/testcontainers/src/s2.ts @@ -0,0 +1,67 @@ +import type { StartedNetwork, StartedTestContainer } from "testcontainers"; +import { GenericContainer, Wait } from "testcontainers"; +import { withCiResourceLimits } from "./utils"; + +/** + * s2-lite 0.40.0 is the first image that emits the tail on the heartbeat ping + * (matching cloud S2 0.25.0), which the caught-up client keys off. Older images + * fire a bare keepalive with no tail, so the digest is pinned to keep the e2e + * deterministic. + */ +const S2_LITE_IMAGE = + "ghcr.io/s2-streamstore/s2:0.40.0@sha256:b26249e2ede0949755f5af8028185dc2bcfc3aa2db21eb9610543d144eb6ee9d"; + +const DEFAULT_BASIN = "trigger-local"; + +function specJson(basin: string): string { + return JSON.stringify({ + basins: [ + { + name: basin, + config: { create_stream_on_append: true, create_stream_on_read: true }, + }, + ], + }); +} + +export interface StartedS2Container { + container: StartedTestContainer; + /** Base URL of the s2-lite HTTP API, e.g. `http://localhost:49xxx`. */ + endpoint: string; + /** The single basin s2-lite is initialised with. */ + basin: string; +} + +/** + * Boots s2-lite (the open-source S2 server) in `lite` mode with a single basin, + * for full-stack realtime session-stream e2e. The spec is copied in rather than + * bind-mounted so the helper carries no external file dependency. + */ +export async function createS2Container( + network?: StartedNetwork, + opts: { basin?: string } = {} +): Promise<StartedS2Container> { + const basin = opts.basin ?? DEFAULT_BASIN; + + let builder = withCiResourceLimits(new GenericContainer(S2_LITE_IMAGE)) + .withExposedPorts(80) + .withCopyContentToContainer([ + { content: specJson(basin), target: "/s2-spec.json", mode: 0o444 }, + ]) + .withCommand(["lite", "--init-file", "/s2-spec.json"]) + .withWaitStrategy(Wait.forLogMessage(/starting plain http server/)) + .withStartupTimeout(60_000); + + if (network) { + builder = builder.withNetwork(network).withNetworkAliases("s2"); + } + + const container = await builder.start(); + const mappedPort = container.getMappedPort(80); + + return { + container, + endpoint: `http://${container.getHost()}:${mappedPort}`, + basin, + }; +} diff --git a/internal-packages/testcontainers/src/webapp.ts b/internal-packages/testcontainers/src/webapp.ts index fc587abd696..22d3485f8fa 100644 --- a/internal-packages/testcontainers/src/webapp.ts +++ b/internal-packages/testcontainers/src/webapp.ts @@ -3,7 +3,9 @@ import { createServer } from "net"; import { delimiter, resolve } from "path"; import { Network } from "testcontainers"; import { PrismaClient } from "@trigger.dev/database"; -import { createPostgresContainer, createRedisContainer } from "./utils"; +import { createPostgresContainer, createRedisContainer, withCiResourceLimits } from "./utils"; +import { createS2Container, type StartedS2Container } from "./s2"; +import { MinIOContainer, type StartedMinIOContainer } from "./minio"; const WEBAPP_ROOT = resolve(__dirname, "../../../apps/webapp"); // pnpm hoists transitive deps to node_modules/.pnpm/node_modules but does NOT symlink them @@ -65,6 +67,13 @@ export interface StartWebappOptions { * behaviour when the loader is short-circuited). */ requirePlugins?: boolean; + + /** + * Extra environment variables merged into the spawned webapp, after the + * defaults so they override them (e.g. the `REALTIME_STREAMS_S2_*` vars for + * session-stream e2e). `NODE_PATH` and the worker-disable vars still win. + */ + extraEnv?: Record<string, string>; } export async function startWebapp( @@ -115,6 +124,7 @@ export async function startWebapp( REDIS_HOST: redis.host, REDIS_PORT: String(redis.port), REDIS_TLS_DISABLED: "true", // all *_REDIS_TLS_DISABLED vars default to this; test Redis has no TLS + ...(options.extraEnv ?? {}), // Disable all background workers. Each worker has its own env var and its own // check idiom ("0" vs "false" vs boolean), so we set all of them explicitly. WORKER_ENABLED: "false", // disables workerQueue.initialize() (checked === "true") @@ -255,3 +265,88 @@ export async function startTestServer(options: StartWebappOptions = {}): Promise return { webapp, prisma: prisma!, databaseUrl: pgUrl!, stop }; } + +export { createS2Container } from "./s2"; +export type { StartedS2Container } from "./s2"; + +export interface SessionStreamTestServer extends TestServer { + s2: StartedS2Container; + minio: StartedMinIOContainer; +} + +/** + * Full-stack session-stream harness: postgres + redis + s2-lite 0.40.0 + the + * real webapp, wired for S2 v2 realtime session streams. The webapp is a host + * process reaching every container over its mapped port, so the S2 endpoint is + * the mapped localhost URL (the docker-network alias is unusable from the host). + */ +export async function startSessionStreamTestServer(): Promise<SessionStreamTestServer> { + const network = await new Network().start(); + + let pgContainer: Awaited<ReturnType<typeof createPostgresContainer>>["container"] | undefined; + let pgUrl: string | undefined; + let redisContainer: Awaited<ReturnType<typeof createRedisContainer>>["container"] | undefined; + let s2: StartedS2Container | undefined; + let minio: StartedMinIOContainer | undefined; + let prisma: PrismaClient | undefined; + let stopWebapp: (() => Promise<void>) | undefined; + let webapp: WebappInstance; + + try { + const pg = await createPostgresContainer(network); + pgContainer = pg.container; + pgUrl = pg.url; + + const { container: rc } = await createRedisContainer({ network }); + redisContainer = rc; + + s2 = await createS2Container(network); + minio = await withCiResourceLimits(new MinIOContainer()).withNetwork(network).start(); + const minioConfig = minio.getConnectionConfig(); + + prisma = new PrismaClient({ datasources: { db: { url: pg.url } } }); + await prisma.$connect(); + + const started = await startWebapp( + pg.url, + { host: rc.getHost(), port: rc.getPort() }, + { + extraEnv: { + REALTIME_STREAMS_S2_ACCESS_TOKEN: "", + REALTIME_STREAMS_S2_BASIN: s2.basin, + REALTIME_STREAMS_S2_ENDPOINT: `${s2.endpoint}/v1`, + REALTIME_STREAMS_S2_SKIP_ACCESS_TOKENS: "true", + REALTIME_STREAMS_DEFAULT_VERSION: "v2", + OBJECT_STORE_BASE_URL: minioConfig.baseUrl, + OBJECT_STORE_BUCKET: "packets", + OBJECT_STORE_ACCESS_KEY_ID: minioConfig.accessKeyId, + OBJECT_STORE_SECRET_ACCESS_KEY: minioConfig.secretAccessKey, + OBJECT_STORE_REGION: minioConfig.region, + }, + } + ); + webapp = started.instance; + stopWebapp = started.stop; + } catch (err) { + await stopWebapp?.().catch(() => {}); + await prisma?.$disconnect().catch(() => {}); + await minio?.stop().catch(() => {}); + await s2?.container.stop().catch(() => {}); + await pgContainer?.stop().catch(() => {}); + await redisContainer?.stop().catch(() => {}); + await network.stop().catch(() => {}); + throw err; + } + + const stop = async () => { + await stopWebapp!().catch((err) => console.error("stopWebapp failed:", err)); + await prisma!.$disconnect().catch((err) => console.error("prisma.$disconnect failed:", err)); + await minio!.stop().catch((err) => console.error("minio.stop failed:", err)); + await s2!.container.stop().catch((err) => console.error("s2.stop failed:", err)); + await pgContainer!.stop().catch((err) => console.error("pgContainer.stop failed:", err)); + await redisContainer!.stop().catch((err) => console.error("redisContainer.stop failed:", err)); + await network.stop().catch((err) => console.error("network.stop failed:", err)); + }; + + return { webapp, prisma: prisma!, databaseUrl: pgUrl!, s2: s2!, minio: minio!, stop }; +} diff --git a/packages/cli-v3/package.json b/packages/cli-v3/package.json index 44c1ec0466d..6d0d90da98f 100644 --- a/packages/cli-v3/package.json +++ b/packages/cli-v3/package.json @@ -95,7 +95,7 @@ "@opentelemetry/resources": "2.7.1", "@opentelemetry/sdk-trace-node": "2.7.1", "@opentelemetry/semantic-conventions": "1.41.1", - "@s2-dev/streamstore": "^0.22.10", + "@s2-dev/streamstore": "^0.25.0", "@trigger.dev/build": "workspace:4.5.9", "@trigger.dev/core": "workspace:4.5.9", "@trigger.dev/schema-to-json": "workspace:4.5.9", diff --git a/packages/core/package.json b/packages/core/package.json index 59cb1c76672..bd890b4e582 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -207,7 +207,7 @@ "@opentelemetry/sdk-trace-base": "2.7.1", "@opentelemetry/sdk-trace-node": "2.7.1", "@opentelemetry/semantic-conventions": "1.41.1", - "@s2-dev/streamstore": "0.22.10", + "@s2-dev/streamstore": "0.25.0", "dequal": "^2.0.3", "eventsource": "^3.0.5", "eventsource-parser": "^3.0.0", diff --git a/packages/core/src/v3/apiClient/runStream.test.ts b/packages/core/src/v3/apiClient/runStream.test.ts index 0dca73779af..3a266f2a918 100644 --- a/packages/core/src/v3/apiClient/runStream.test.ts +++ b/packages/core/src/v3/apiClient/runStream.test.ts @@ -27,6 +27,22 @@ describe("SSEStreamSubscription retry behavior", () => { }); } + /** + * A response.body that emits one SSE event then stays open (never closes), + * so the connection sits mid-read until the fetch signal aborts. + */ + function makeOpenSSEResponse() { + const body = new ReadableStream<Uint8Array>({ + start(controller) { + controller.enqueue(new TextEncoder().encode(`id: 1\ndata: {"hello":1}\n\n`)); + }, + }); + return new Response(body, { + status: 200, + headers: { "Content-Type": "text/event-stream", "X-Stream-Version": "v1" }, + }); + } + // Drain a ReadableStream<SSEStreamPart> until it closes or errors. // Returns received chunks plus terminal state. async function drain(stream: ReadableStream<{ id: string; chunk: unknown }>) { @@ -74,6 +90,30 @@ describe("SSEStreamSubscription retry behavior", () => { expect(result.chunks).toHaveLength(1); }); + it("does not reconnect after the consumer cancels the returned stream", async () => { + let attempts = 0; + globalThis.fetch = vi.fn().mockImplementation(async () => { + attempts++; + return makeOpenSSEResponse(); + }); + + const sub = new SSEStreamSubscription("http://example.test/sse", { + retryDelayMs: 1, + maxRetryDelayMs: 5, + maxRetries: 10, + }); + + const stream = await sub.subscribe(); + const reader = stream.getReader(); + const first = await reader.read(); + expect(first.done).toBe(false); + + await reader.cancel(); + await new Promise((r) => setTimeout(r, 50)); + + expect(attempts).toBe(1); + }); + it("caps the exponential backoff at maxRetryDelayMs", async () => { let attempts = 0; const callTimes: number[] = []; diff --git a/packages/core/src/v3/apiClient/runStream.ts b/packages/core/src/v3/apiClient/runStream.ts index 5bd7986c86a..acf3f5361df 100644 --- a/packages/core/src/v3/apiClient/runStream.ts +++ b/packages/core/src/v3/apiClient/runStream.ts @@ -184,6 +184,12 @@ export type SSEStreamPart<TChunk = unknown> = { headers?: Array<[string, string]>; }; +/** + * Internal item flowing from the decode transform to the consumer-facing + * stream. + */ +type PumpItem = { type: "part"; part: SSEStreamPart }; + // Real implementation for production export class SSEStreamSubscription implements StreamSubscription { private lastEventId: string | undefined; @@ -197,6 +203,8 @@ export class SSEStreamSubscription implements StreamSubscription { private nonRetryableStatuses: ReadonlySet<number>; private retryNowController: AbortController | null = null; private internalAbort: AbortController | null = null; + private cancelledByConsumer = false; + private completeNotified = false; constructor( private url: string, @@ -278,22 +286,64 @@ export class SSEStreamSubscription implements StreamSubscription { this.retryNowController?.abort(); } + /** Fire `onComplete` at most once, even if both the drain and cancel paths reach it. */ + private notifyComplete(): void { + if (this.completeNotified) return; + this.completeNotified = true; + this.options.onComplete?.(); + } + + /** + * The transport pumps decoded records into an internal stream; the returned + * stream drains it on demand. + */ async subscribe(): Promise<ReadableStream<SSEStreamPart>> { // eslint-disable-next-line no-this-alias const self = this; - return new ReadableStream({ + const internal = new ReadableStream<PumpItem>({ async start(controller) { await self.connectStream(controller); }, cancel() { - self.options.onComplete?.(); + self.cancelledByConsumer = true; + self.internalAbort?.abort(); + self.retryNowController?.abort(); }, }); + const internalReader = internal.getReader(); + + return new ReadableStream<SSEStreamPart>( + { + async pull(controller) { + let result: ReadableStreamReadResult<PumpItem>; + try { + result = await internalReader.read(); + } catch (err) { + controller.error(err); + return; + } + if (result.done) { + self.notifyComplete(); + try { + controller.close(); + } catch {} + return; + } + controller.enqueue(result.value.part); + }, + cancel(reason) { + self.cancelledByConsumer = true; + self.notifyComplete(); + internalReader.cancel(reason).catch(() => {}); + }, + }, + { highWaterMark: 0 } + ); } private async connectStream( - controller: ReadableStreamDefaultController<SSEStreamPart> + controller: ReadableStreamDefaultController<PumpItem> ): Promise<void> { // Two abort sources flow through `internalAbort.signal`: // - this.options.signal: caller cancel — bypass retry, exit cleanly. @@ -386,7 +436,7 @@ export class SSEStreamSubscription implements StreamSubscription { .pipeThrough(new TextDecoderStream()) .pipeThrough(new EventSourceParserStream()) .pipeThrough( - new TransformStream<EventSourceMessage, SSEStreamPart>({ + new TransformStream<EventSourceMessage, PumpItem>({ transform: (chunk, chunkController) => { if (streamVersion === "v1") { if (chunk.id) { @@ -394,9 +444,12 @@ export class SSEStreamSubscription implements StreamSubscription { } const timestamp = parseRedisStreamIdTimestamp(chunk.id); chunkController.enqueue({ - id: chunk.id ?? "unknown", - chunk: safeParseJSON(chunk.data), - timestamp, + type: "part", + part: { + id: chunk.id ?? "unknown", + chunk: safeParseJSON(chunk.data), + timestamp, + }, }); } else { if (chunk.event === "batch") { @@ -407,6 +460,7 @@ export class SSEStreamSubscription implements StreamSubscription { timestamp: number; headers?: Array<[string, string]>; }>; + tail?: { seq_num: number; timestamp: number }; }; if (!data || !Array.isArray(data.records)) return; @@ -438,10 +492,13 @@ export class SSEStreamSubscription implements StreamSubscription { rememberSeen(parsedBody.id); } chunkController.enqueue({ - id: record.seq_num.toString(), - chunk: parsedBody?.data, - timestamp: record.timestamp, - headers: record.headers ?? [], + type: "part", + part: { + id: record.seq_num.toString(), + chunk: parsedBody?.data, + timestamp: record.timestamp, + headers: record.headers ?? [], + }, }); } } @@ -458,16 +515,16 @@ export class SSEStreamSubscription implements StreamSubscription { if (done) { reader.releaseLock(); + this.notifyComplete(); controller.close(); - this.options.onComplete?.(); return; } if (this.options.signal?.aborted) { reader.cancel(); reader.releaseLock(); + this.notifyComplete(); controller.close(); - this.options.onComplete?.(); return; } @@ -479,10 +536,10 @@ export class SSEStreamSubscription implements StreamSubscription { throw error; } } catch (error) { - if (this.options.signal?.aborted) { + if (this.options.signal?.aborted || this.cancelledByConsumer) { // User cancel — exit cleanly, don't retry. + this.notifyComplete(); controller.close(); - this.options.onComplete?.(); return; } @@ -505,9 +562,9 @@ export class SSEStreamSubscription implements StreamSubscription { controller: ReadableStreamDefaultController, error?: Error ): Promise<void> { - if (this.options.signal?.aborted) { + if (this.options.signal?.aborted || this.cancelledByConsumer) { + this.notifyComplete(); controller.close(); - this.options.onComplete?.(); return; } @@ -547,9 +604,8 @@ export class SSEStreamSubscription implements StreamSubscription { }); this.retryNowController = null; - if (this.options.signal?.aborted) { + if (this.options.signal?.aborted || this.cancelledByConsumer) { controller.close(); - this.options.onComplete?.(); return; } diff --git a/packages/core/src/v3/realtimeStreams/streamsWriterV2.ts b/packages/core/src/v3/realtimeStreams/streamsWriterV2.ts index c8fcb249e18..4a25eb86a4b 100644 --- a/packages/core/src/v3/realtimeStreams/streamsWriterV2.ts +++ b/packages/core/src/v3/realtimeStreams/streamsWriterV2.ts @@ -183,10 +183,16 @@ export class StreamsWriterV2<T = any> implements StreamsWriter { const lastAcked = session.lastAckedPosition(); if (lastAcked?.end) { - this.lastSeqNum = lastAcked.end.seqNum; - this.log( - `[S2MetadataStream] Written ${this.lastSeqNum} records, ending at seqNum=${this.lastSeqNum}` - ); + /** + * S2's ack `end.seqNum` is exclusive: the seq AFTER the last record + * written (equal to the tail). Report the last record's own seq + * (`end - 1`) as `lastEventId` so a resume subscribe, which reads + * from `lastEventId + 1`, lands on the next record instead of + * skipping one. Matches the inclusive seq the one-shot control + * writer returns. + */ + this.lastSeqNum = lastAcked.end.seqNum - 1; + this.log(`[S2MetadataStream] Wrote through seqNum=${this.lastSeqNum}`); } } catch (error) { if (this.aborted) { diff --git a/packages/core/src/v3/test/index.ts b/packages/core/src/v3/test/index.ts index 402f618c01b..ec7ba85d913 100644 --- a/packages/core/src/v3/test/index.ts +++ b/packages/core/src/v3/test/index.ts @@ -7,3 +7,10 @@ export { TestInputStreamManager } from "./test-input-stream-manager.js"; export { TestRealtimeStreamsManager } from "./test-realtime-streams-manager.js"; export { TestRunMetadataManager } from "./test-run-metadata-manager.js"; export { TestSessionStreamManager } from "./test-session-stream-manager.js"; +export { StandardSessionStreamManager } from "../sessionStreams/manager.js"; +export type { SessionStreamManager } from "../sessionStreams/types.js"; +export { + SessionWaitpointBackend, + TestRuntimeManager, + installSessionWaitpointBackend, +} from "./session-waitpoint-backend.js"; diff --git a/packages/core/src/v3/test/mock-task-context.ts b/packages/core/src/v3/test/mock-task-context.ts index 0da633c6a7e..5fbe1957613 100644 --- a/packages/core/src/v3/test/mock-task-context.ts +++ b/packages/core/src/v3/test/mock-task-context.ts @@ -9,10 +9,11 @@ import { runtime } from "../runtime-api.js"; import { StandardLocalsManager } from "../locals/manager.js"; import { StandardLifecycleHooksManager } from "../lifecycleHooks/manager.js"; import { NoopRuntimeManager } from "../runtime/noopRuntimeManager.js"; +import type { RuntimeManager } from "../runtime/manager.js"; import { unregisterGlobal } from "../utils/globals.js"; import type { ServerBackgroundWorker, TaskRunContext } from "../schemas/index.js"; import type { LocalsKey } from "../locals/types.js"; -import type { SessionChannelIO } from "../sessionStreams/types.js"; +import type { SessionChannelIO, SessionStreamManager } from "../sessionStreams/types.js"; import { TestInputStreamManager } from "./test-input-stream-manager.js"; import { TestRealtimeStreamsManager } from "./test-realtime-streams-manager.js"; import { TestRunMetadataManager } from "./test-run-metadata-manager.js"; @@ -45,6 +46,20 @@ export type MockTaskContextOptions = { worker?: Partial<ServerBackgroundWorker>; /** Whether this is a warm start. */ isWarmStart?: boolean; + /** + * Session-streams manager installed as the `session-streams` global. Defaults + * to an in-memory {@link TestSessionStreamManager}. Pass a real + * `StandardSessionStreamManager` (with an ApiClient pointed at a running + * webapp) to drive the task's `.in`/`.out` against real streams. + */ + sessionStreamManager?: SessionStreamManager; + /** + * Runtime manager installed as the `runtime` global. Defaults to a + * {@link NoopRuntimeManager}. Pass a `TestRuntimeManager` (wired to a + * {@link SessionWaitpointBackend}) to make `session.in.wait()` suspend and + * resume in place against real streams, without the run-engine. + */ + runtimeManager?: RuntimeManager; }; /** @@ -215,11 +230,11 @@ export async function runInMockTaskContext<T>( const localsManager = new StandardLocalsManager(); const lifecycleManager = new StandardLifecycleHooksManager(); - const runtimeManager = new NoopRuntimeManager(); + const runtimeManager = options?.runtimeManager ?? new NoopRuntimeManager(); const metadataManager = new TestRunMetadataManager(); const inputManager = new TestInputStreamManager(); const outputManager = new TestRealtimeStreamsManager(); - const sessionStreamManager = new TestSessionStreamManager(); + const sessionStreamManager = options?.sessionStreamManager ?? new TestSessionStreamManager(); // Unregister any previously-installed managers so `setGlobal*` wins — // `registerGlobal` returns false silently if an entry already exists. @@ -263,8 +278,16 @@ export async function runInMockTaskContext<T>( sessions: { in: { send: (sessionId, data, io = "in") => - sessionStreamManager.__sendFromTest(sessionId, io, data), - close: (sessionId, io = "in") => sessionStreamManager.__closeFromTest(sessionId, io), + sessionStreamManager instanceof TestSessionStreamManager + ? sessionStreamManager.__sendFromTest(sessionId, io, data) + : Promise.reject( + new Error("drivers.sessions.in.send requires the default TestSessionStreamManager") + ), + close: (sessionId, io = "in") => { + if (sessionStreamManager instanceof TestSessionStreamManager) { + sessionStreamManager.__closeFromTest(sessionId, io); + } + }, }, }, ctx, @@ -286,7 +309,9 @@ export async function runInMockTaskContext<T>( localsManager.reset(); inputManager.reset(); outputManager.reset(); - sessionStreamManager.reset(); + if (sessionStreamManager instanceof TestSessionStreamManager) { + sessionStreamManager.reset(); + } metadataManager.reset(); } } diff --git a/packages/core/src/v3/test/session-waitpoint-backend.ts b/packages/core/src/v3/test/session-waitpoint-backend.ts new file mode 100644 index 00000000000..8cae877f54e --- /dev/null +++ b/packages/core/src/v3/test/session-waitpoint-backend.ts @@ -0,0 +1,267 @@ +import { ApiClient } from "../apiClient/index.js"; +import { WaitpointId } from "../isomorphic/friendlyId.js"; +import { NoopRuntimeManager } from "../runtime/noopRuntimeManager.js"; +import type { + CreateSessionStreamWaitpointRequestBody, + CreateSessionStreamWaitpointResponseBody, + WaitForWaitpointTokenResponseBody, +} from "../schemas/api.js"; +import type { WaitpointTokenResult } from "../schemas/common.js"; + +type PendingWait = { + session: string; + io: "in" | "out"; + lastSeqNum?: number; + timeout?: string; + abort: AbortController; +}; + +const TIMED_OUT = Symbol("session-waitpoint-timeout"); + +function parseTimeoutMs(timeout: string | undefined): number | undefined { + if (!timeout) { + return undefined; + } + const match = /^(\d+)(ms|s|m|h)$/.exec(timeout.trim()); + if (!match) { + return undefined; + } + const value = Number(match[1]); + switch (match[2]) { + case "ms": + return value; + case "s": + return value * 1_000; + case "m": + return value * 60_000; + default: + return value * 3_600_000; + } +} + +/** + * In-process stand-in for the run-engine's session-stream waitpoint machinery. + * + * `session.in.wait()` suspends a run on a waitpoint; in production the run-engine + * completes that waitpoint (and the supervisor resumes the run) when the next + * `.in` record arrives. This backend reproduces the task-observable half in one + * process: `register()` mints a waitpoint id when the SDK creates the waitpoint, + * and `wait()` opens its own short-lived tail on the session channel and resolves + * with the next record. That record is handed back through `TestRuntimeManager` + * exactly as the real runtime hands back a completed waitpoint's output, so the + * same `run()` invocation continues in place (the faithful dev / deployed + * task-observable semantics: same promise, same record, no re-invocation). + * + * It deliberately does not model server-side waitpoint bookkeeping, process + * checkpoint/restore, or client-vs-server timeout, none of which are visible to + * task code on the resume path. + */ +export class SessionWaitpointBackend { + private readonly pending = new Map<string, PendingWait>(); + private disabled = false; + + constructor(private readonly apiClient: ApiClient) {} + + register( + body: CreateSessionStreamWaitpointRequestBody + ): CreateSessionStreamWaitpointResponseBody { + const waitpointId = WaitpointId.generate().friendlyId; + this.pending.set(waitpointId, { + session: body.session, + io: body.io, + lastSeqNum: body.lastSeqNum, + timeout: body.timeout, + abort: new AbortController(), + }); + return { waitpointId, isCached: false }; + } + + async wait(waitpointFriendlyId: string): Promise<WaitpointTokenResult> { + if (this.disabled) { + return { + ok: false, + output: JSON.stringify({ message: "Session waitpoint backend disabled" }), + outputType: "application/json", + }; + } + const pending = this.pending.get(waitpointFriendlyId); + if (!pending) { + return { ok: true }; + } + + const timeoutMs = parseTimeoutMs(pending.timeout); + let timer: ReturnType<typeof setTimeout> | undefined; + + try { + const recordPromise = this.readNextRecord(pending); + const result = + timeoutMs === undefined + ? await recordPromise + : await Promise.race([ + recordPromise, + new Promise<typeof TIMED_OUT>((resolve) => { + timer = setTimeout(() => resolve(TIMED_OUT), timeoutMs); + }), + ]); + + if (result === TIMED_OUT) { + pending.abort.abort(); + return { + ok: false, + output: JSON.stringify({ message: "Timed out" }), + outputType: "application/json", + }; + } + + const output = typeof result === "string" ? result : JSON.stringify(result); + return { ok: true, output, outputType: "application/json" }; + } catch { + return { + ok: false, + output: JSON.stringify({ message: "Session stream wait ended before a record arrived" }), + outputType: "application/json", + }; + } finally { + if (timer) { + clearTimeout(timer); + } + this.pending.delete(waitpointFriendlyId); + } + } + + disable(): void { + this.disabled = true; + for (const pending of this.pending.values()) { + pending.abort.abort(); + } + this.pending.clear(); + } + + /** + * Opens an independent tail on the session channel starting after the + * caller's last-seen seq and resolves with the next record. For `.in` the + * tail yields the raw appended JSON string (already `JSON.stringify(chunk)`), + * which {@link wait} passes straight to the packet parser so it round-trips + * to the same object `session.in.once()` returns. + */ + private async readNextRecord(pending: PendingWait): Promise<unknown> { + const lastEventId = + pending.lastSeqNum !== undefined && pending.lastSeqNum >= 0 + ? String(pending.lastSeqNum) + : undefined; + + const stream = await this.apiClient.subscribeToSessionStream(pending.session, pending.io, { + lastEventId, + signal: pending.abort.signal, + timeoutInSeconds: 120, + }); + + const reader = stream.getReader(); + try { + const { value, done } = await reader.read(); + if (done) { + throw new Error("session stream closed"); + } + return value; + } finally { + await reader.cancel().catch(() => {}); + pending.abort.abort(); + } + } +} + +/** + * A {@link NoopRuntimeManager} whose `waitForWaitpoint` resolves in place from a + * {@link SessionWaitpointBackend} instead of hanging. Install it as the `runtime` + * global via `runInMockTaskContext({ runtimeManager })`. + */ +export class TestRuntimeManager extends NoopRuntimeManager { + constructor(private readonly backend: SessionWaitpointBackend) { + super(); + } + + override waitForWaitpoint(params: { + waitpointFriendlyId: string; + finishDate?: Date; + }): Promise<WaitpointTokenResult> { + return this.backend.wait(params.waitpointFriendlyId); + } +} + +let activeBackend: SessionWaitpointBackend | undefined; +let patchDepth = 0; +let originalCreate: ApiClient["createSessionStreamWaitpoint"] | undefined; +let originalWait: ApiClient["waitForWaitpointToken"] | undefined; + +/** + * Route the two waitpoint apiClient calls through an in-process backend for the + * duration of a harness run, and hand back a {@link TestRuntimeManager} to + * install as the `runtime` global. + * + * `apiClientManager.clientOrThrow()` builds a fresh `ApiClient` per call, so + * there is no instance to swap; the two methods are patched on the prototype + * (ref-counted, restored by `restore()`). `createSessionStreamWaitpoint` + * registers a pending wait and returns a real-shaped `{ waitpointId, isCached }`; + * `waitForWaitpointToken` returns `{ success: true }` exactly like the real route, + * which only creates the block edge and returns synchronously. + */ +export function installSessionWaitpointBackend(apiClient: ApiClient): { + backend: SessionWaitpointBackend; + runtimeManager: TestRuntimeManager; + restore: () => void; +} { + const backend = new SessionWaitpointBackend(apiClient); + const previousBackend = activeBackend; + activeBackend = backend; + + if (patchDepth === 0) { + originalCreate = ApiClient.prototype.createSessionStreamWaitpoint; + originalWait = ApiClient.prototype.waitForWaitpointToken; + + ApiClient.prototype.createSessionStreamWaitpoint = function ( + this: ApiClient, + runFriendlyId: string, + body: CreateSessionStreamWaitpointRequestBody + ): Promise<CreateSessionStreamWaitpointResponseBody> { + if (activeBackend) { + return Promise.resolve(activeBackend.register(body)); + } + return originalCreate!.call(this, runFriendlyId, body); + } as unknown as ApiClient["createSessionStreamWaitpoint"]; + + ApiClient.prototype.waitForWaitpointToken = function ( + this: ApiClient + ): Promise<WaitForWaitpointTokenResponseBody> { + if (activeBackend) { + return Promise.resolve({ success: true }); + } + // eslint-disable-next-line prefer-rest-params + return (originalWait as ApiClient["waitForWaitpointToken"]).apply(this, arguments as never); + } as unknown as ApiClient["waitForWaitpointToken"]; + } + + patchDepth += 1; + let restored = false; + + return { + backend, + runtimeManager: new TestRuntimeManager(backend), + restore() { + if (restored) { + return; + } + restored = true; + backend.disable(); + if (activeBackend === backend) { + activeBackend = previousBackend; + } + patchDepth -= 1; + if (patchDepth === 0 && originalCreate && originalWait) { + ApiClient.prototype.createSessionStreamWaitpoint = originalCreate; + ApiClient.prototype.waitForWaitpointToken = originalWait; + originalCreate = undefined; + originalWait = undefined; + } + }, + }; +} diff --git a/packages/trigger-sdk/src/v3/ai.ts b/packages/trigger-sdk/src/v3/ai.ts index 65ec72f4308..844d506079b 100644 --- a/packages/trigger-sdk/src/v3/ai.ts +++ b/packages/trigger-sdk/src/v3/ai.ts @@ -6063,8 +6063,23 @@ function chatAgent< ); } + /** + * A retry (e.g. OOM) or continuation whose boot recovery restored an + * in-flight `.in` message dispatches it as the first turn. The recovery + * block already advanced the `.in` cursor past the recovered message, + * so the preload wait below would otherwise strand it (the run would + * sit waiting for a "first message" that already arrived). `onPreload` + * is a one-time start-of-run hook and is deliberately not re-run on a + * recovered/continuation turn. + */ + let dispatchedRecoveredFirstTurn = false; + if (preloaded && bootInjectedQueue.length > 0) { + currentWirePayload = bootInjectedQueue.shift()!; + dispatchedRecoveredFirstTurn = true; + } + // Handle preloaded runs — fire onPreload, then wait for the first real message - if (preloaded) { + if (preloaded && !dispatchedRecoveredFirstTurn) { if (activeSpan) { activeSpan.setAttribute("chat.preloaded", true); } diff --git a/packages/trigger-sdk/src/v3/chat-client.ts b/packages/trigger-sdk/src/v3/chat-client.ts index 9836346fad7..f632f5e89d5 100644 --- a/packages/trigger-sdk/src/v3/chat-client.ts +++ b/packages/trigger-sdk/src/v3/chat-client.ts @@ -576,10 +576,26 @@ export class AgentChat<TAgent = unknown> { } } - /** Reconnect to the response stream (e.g. after a disconnect). */ + /** + * Reconnect to the response stream to resume an already-established session + * after a disconnect. Requests the caught-up settle probe (`X-Peek-Settled`) + * so a resumed stream already at a turn-complete tail closes promptly instead + * of holding the full SSE window, mirroring the browser transport's + * `reconnectToStream`. + * + * Do not call this immediately after `sendMessage()` to read a fresh turn: the + * settle probe can race the newly-triggered turn's first record, see the prior + * turn's `turn-complete` tail, and hand back an empty stream (unlike the + * browser transport, there is no auto-resubscribe). `sendMessage()` already + * returns the turn's stream; use `reconnect()` only to resume an idle or + * mid-turn stream. + */ async reconnect(abortSignal?: AbortSignal): Promise<ReadableStream<UIMessageChunk> | null> { if (!this.state.started) return null; - return this.subscribeToSessionStream(abortSignal, { sendStopOnAbort: false }); + return this.subscribeToSessionStream(abortSignal, { + sendStopOnAbort: false, + peekSettled: true, + }); } // ─── Private ─────────────────────────────────────────────────── @@ -681,7 +697,7 @@ export class AgentChat<TAgent = unknown> { private subscribeToSessionStream( abortSignal: AbortSignal | undefined, - options?: { sendStopOnAbort?: boolean } + options?: { sendStopOnAbort?: boolean; peekSettled?: boolean } ): ReadableStream<UIMessageChunk> { const state = this.state; const accessToken = apiClientManager.accessToken ?? ""; @@ -743,6 +759,7 @@ export class AgentChat<TAgent = unknown> { ...(apiClientManager.branchName ? { "x-trigger-branch": apiClientManager.branchName } : {}), + ...(options?.peekSettled ? { "X-Peek-Settled": "1" } : {}), }, signal: combinedSignal, timeoutInSeconds: this.streamTimeoutSeconds, diff --git a/packages/trigger-sdk/src/v3/chat.ts b/packages/trigger-sdk/src/v3/chat.ts index 9dddac1c0eb..85f2ac75156 100644 --- a/packages/trigger-sdk/src/v3/chat.ts +++ b/packages/trigger-sdk/src/v3/chat.ts @@ -1725,6 +1725,8 @@ export class TriggerChatTransport implements ChatTransport<UIMessage> { ); }) as typeof fetch) : undefined; + let sawFirstChunk = false; + const connectSseOnce = async (token: string) => { const subscription = new SSEStreamSubscription(streamUrl, { headers: { @@ -1750,7 +1752,7 @@ export class TriggerChatTransport implements ChatTransport<UIMessage> { reader.releaseLock(); return null; } - return { reader, primed: first.value }; + return { reader, primed: first.value, subscription }; } catch (readErr) { reader.releaseLock(); throw readErr; @@ -1798,7 +1800,6 @@ export class TriggerChatTransport implements ChatTransport<UIMessage> { lastEventId: state.lastEventId, messageId: this.lastTurnSends.get(chatId)?.messageId, }); - let sawFirstChunk = false; while (true) { let value: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6c8be5550d8..f37e3250982 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -444,8 +444,8 @@ importers: specifier: 2.17.5 version: 2.17.5(typescript@6.0.3) '@s2-dev/streamstore': - specifier: ^0.22.10 - version: 0.22.10(supports-color@10.0.0) + specifier: ^0.25.0 + version: 0.25.0(supports-color@10.0.0) '@sentry/remix': specifier: 9.46.0 version: 9.46.0(patch_hash=146126b032581925294aaed63ab53ce3f5e0356a755f1763d7a9a76b9846943b)(@remix-run/node@2.17.5(typescript@6.0.3))(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3))(encoding@0.1.13)(react@18.3.1) @@ -792,6 +792,9 @@ importers: '@internal/testcontainers': specifier: workspace:* version: link:../../internal-packages/testcontainers + '@playwright/test': + specifier: ^1.36.2 + version: 1.37.0 '@remix-run/dev': specifier: 2.17.5 version: 2.17.5(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/serve@2.17.5(typescript@6.0.3))(@types/node@24.13.3)(bufferutil@4.0.9)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.20.6)(typescript@6.0.3)(vite@6.4.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.20.6)(yaml@2.9.0))(yaml@2.9.0) @@ -1529,8 +1532,8 @@ importers: specifier: 1.41.1 version: 1.41.1 '@s2-dev/streamstore': - specifier: ^0.22.10 - version: 0.22.10(supports-color@10.0.0) + specifier: ^0.25.0 + version: 0.25.0(supports-color@10.0.0) '@trigger.dev/build': specifier: workspace:4.5.9 version: link:../build @@ -1800,8 +1803,8 @@ importers: specifier: 1.41.1 version: 1.41.1 '@s2-dev/streamstore': - specifier: 0.22.10 - version: 0.22.10(supports-color@10.0.0) + specifier: 0.25.0 + version: 0.25.0(supports-color@10.0.0) dequal: specifier: ^2.0.3 version: 2.0.3 @@ -6794,8 +6797,8 @@ packages: cpu: [x64] os: [win32] - '@s2-dev/streamstore@0.22.10': - resolution: {integrity: sha512-dtm+oFHVE8szINwOUoNQdx9xpGSJOrcAEvsxspPFvomjYKGnmhIRmU4OX8o6kxcPoiK76S1tPeU0smjZdmOngA==} + '@s2-dev/streamstore@0.25.0': + resolution: {integrity: sha512-oB8OJObT/s2Q68Tr01NLfDnQPFJp/Kn2pP3seWybgP7VrjQ+j1efXReOsyVCDsUBVMxkyPlzt75o/+GuL5Q3vg==} '@sec-ant/readable-stream@0.4.1': resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==} @@ -21175,7 +21178,7 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.60.1': optional: true - '@s2-dev/streamstore@0.22.10(supports-color@10.0.0)': + '@s2-dev/streamstore@0.25.0(supports-color@10.0.0)': dependencies: '@protobuf-ts/runtime': 2.11.1 debug: 4.4.3(supports-color@10.0.0) From cb9aefd49b9e338dc97d78bc2cca9e9634ea5a4b Mon Sep 17 00:00:00 2001 From: Matt Aitken <matt@mattaitken.com> Date: Sat, 1 Aug 2026 14:17:27 +0100 Subject: [PATCH 55/67] fix(hosting): deploy ClickHouse from the official image instead of Bitnami (#4249) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Self-hosted deployments now run ClickHouse from the official [`clickhouse/clickhouse-server`](https://hub.docker.com/r/clickhouse/clickhouse-server) image instead of `bitnamilegacy/clickhouse`. Bitnami's free image catalog is EOL and the frozen legacy archive tops out at ClickHouse 25.7.5, below the 25.8 minimum the platform requires since v4.5.0, which broke every ClickHouse insert on chart-bundled deployments. Both stacks now default to 26.2, the same version the platform is developed and tested against. Existing deployments keep their ClickHouse data with no manual migration. Fixes #4197. ## Details **Docker Compose**: the `clickhouse` service uses the official image with its native env vars, plus the recommended `nofile` ulimits. It reuses the same named volume as before: a `data-paths.xml` config override points ClickHouse at the `data/` subdirectory of the volume, which is exactly the layout the Bitnami image used, so old volumes work in place (including SQL-created users) and fresh installs get the identical layout. The service follows the required-secrets model: `CLICKHOUSE_PASSWORD` must be set, matching the other services. **Helm chart**: the Bitnami ClickHouse subchart is replaced by a chart-owned single-node StatefulSet and Service running the official image (non-root, HTTP `/ping` probes, config overrides mounted into `config.d`, and the same `data-paths.xml` layout compatibility). On upgrade, the chart automatically adopts the data PVC left behind by the old subchart (`data-<release>-clickhouse-shard0-0`) via `lookup`, and `fsGroup` relabeling handles the uid change on first mount. Both the ClickHouse server and the webapp read the password from the same chart-managed datastore secret (auto-generated and retained across upgrades), so the server credential and the app's connection URL always match. Existing `clickhouse.*` values keep working: `auth` (including `existingSecret`/`existingSecretKey`), `persistence` (including `global.storageClass`), `resources`, `secure`, `external.*`, `configdFiles`, and now `nodeSelector`/`tolerations`/`affinity`. Bitnami-only keys (`shards`, `replicaCount`, `keeper`, `resourcesPreset`) are gone; default `resources` requests/limits match what the old preset applied. The docs state the 25.8 minimum for bring-your-own ClickHouse. ## Upgrade caveats An adversarial review of the upgrade path found a few cohorts that need awareness (all documented): - **GitOps tools that render with `helm template`** (no cluster access): PVC auto-detection can't run, so `clickhouse.persistence.existingClaim` must be set to the old PVC name or ClickHouse starts on a fresh empty volume. Documented in the values file and the Kubernetes self-hosting docs. Tools that run real helm installs (e.g. Flux) adopt automatically. - **A pinned `CLICKHOUSE_IMAGE_TAG`** pointing at a Bitnami tag must be updated to an official image tag; documented in the Docker self-hosting docs. - **Storage without `fsGroup` support** (NFS, hostPath): set `clickhouse.volumePermissions.enabled: true` for a one-time ownership-fixing init container. - **Rollback is not automatic**: once the official image has run, file ownership changes and the Bitnami image can no longer read the volume without a manual chown, and ClickHouse does not support downgrades across the version gap. ## Verification - Full upgrade simulation for Compose, twice (before and after rebasing onto the required-secrets release): booted the ClickHouse service from the old compose file on `main` (Bitnami), wrote thousands of rows, then brought the same project up with this branch's compose file. The official 26.2 server came up healthy on the same volume with all rows intact, SQL-created users working, and writes succeeding. - Adoption scenarios tested against real containers: old volume + root entrypoint (Compose), old volume owned by the Bitnami uid + non-root 101 with fsGroup-style group permissions (Kubernetes), and fresh volumes for both. - `helm lint`, `helm template` (default values, `existingClaim` set, external ClickHouse, volumePermissions/scheduling toggles, and the production example) and kubeconform all pass, mirroring the release CI steps. The rendered webapp Deployment and ClickHouse StatefulSet resolve to the same datastore secret key. - Inserts using `input_format_json_infer_array_of_dynamic_from_array_of_different_types` (the setting that fails on 25.7.5) succeed on the upgraded volume. ## Upgrade preflight and docs A production upgrade report on this branch surfaced two hazards that predate this PR — both landed in chart 4.5.6 (#4316) — so they are fixed here rather than left for the next person to hit. **`secrets.existingSecret` gained two required keys.** The webapp started reading `PROVIDER_SECRET` and `COORDINATOR_SECRET`, and when `existingSecret` is set the chart generates nothing, so a missing key only surfaced as a `CreateContainerConfigError` partway through the webapp rollout. The pre-install/pre-upgrade validation now looks the Secret up and fails with the complete list of missing keys, leaving the running release untouched. It is skipped under `helm template` and client-side dry-run, where `lookup` cannot read the cluster. **Bundled datastore credentials moved into the chart-managed Secret** (`<release>-clickhouse`/`admin-password` → `trigger-datastore`/`clickhouse-admin-password`). The chart wires both ends itself, but consumers outside it — maintenance CronJobs, Grafana datasources, secret syncs — have to be repointed. A new `## Upgrading` section in the Kubernetes docs carries the old→new mapping, the two new keys, and a pointer to the ClickHouse image notes. The existingSecret key list in the docs also named `OBJECT_STORE_ACCESS_KEY_ID`/`OBJECT_STORE_SECRET_ACCESS_KEY`, which are env var names rather than keys the chart reads; corrected to the real key names and the condition under which they apply. Verified on a throwaway kind cluster with `--dry-run=server`: a pre-4.5.6 Secret fails with both key names listed, the documented `kubectl patch` clears it, and default values, `existingClaim`, external ClickHouse, volumePermissions/scheduling and the production example all still render. A real `helm install` followed by an upgrade against an incomplete Secret aborts with the release still at revision 1 and `deployed`. `helm lint`, the CI render and kubeconform (59 resources, 0 invalid) pass. --------- Co-authored-by: nicktrn <55853254+nicktrn@users.noreply.github.com> --- docs/self-hosting/docker.mdx | 11 + docs/self-hosting/kubernetes.mdx | 97 ++++++- hosting/docker/.env.example | 2 +- hosting/docker/clickhouse/data-paths.xml | 17 ++ hosting/docker/webapp/docker-compose.yml | 22 +- hosting/k8s/helm/Chart.lock | 7 +- hosting/k8s/helm/Chart.yaml | 4 - hosting/k8s/helm/templates/_helpers.tpl | 60 ++++- hosting/k8s/helm/templates/clickhouse.yaml | 242 ++++++++++++++++++ .../helm/templates/tests/test-clickhouse.yaml | 12 +- .../templates/validate-external-config.yaml | 29 +++ hosting/k8s/helm/templates/webapp.yaml | 6 +- .../k8s/helm/values-production-example.yaml | 4 +- hosting/k8s/helm/values.yaml | 100 +++++++- 14 files changed, 561 insertions(+), 52 deletions(-) create mode 100644 hosting/docker/clickhouse/data-paths.xml create mode 100644 hosting/k8s/helm/templates/clickhouse.yaml diff --git a/docs/self-hosting/docker.mdx b/docs/self-hosting/docker.mdx index 6c8de292f17..065b4ad55da 100644 --- a/docs/self-hosting/docker.mdx +++ b/docs/self-hosting/docker.mdx @@ -362,6 +362,17 @@ TRIGGER_IMAGE_TAG=v4.5.0 We patch the latest released version line only, so keep an eye on new releases to receive security fixes. See [Security & vulnerability reporting](/self-hosting/security). +You can also lock the versions of the bundled services, for example with `CLICKHOUSE_IMAGE_TAG`. If you do, or if you bring your own ClickHouse via `CLICKHOUSE_URL`, note that Trigger.dev requires ClickHouse 25.8 or newer. + +<Note> + The bundled ClickHouse now uses the official `clickhouse/clickhouse-server` image. Your existing + data volume carries over automatically. If you previously pinned `CLICKHOUSE_IMAGE_TAG` to a + Bitnami tag (for example `25.7.5-debian-12-r0`), update it to an official image tag such as + `26.2` — Bitnami tags don't exist in the official repository. Note the switch is one-way: the + official image takes ownership of the data files, so rolling back to the Bitnami image requires + manually restoring their previous owner (`chown -R 1001:1001` on the volume). +</Note> + <Note> Trigger.dev 4.5.0 is the last version we officially support for running v3 (SDK v3) tasks. If you still have v3 tasks, pin `TRIGGER_IMAGE_TAG` to exactly `v4.5.0` or [migrate to diff --git a/docs/self-hosting/kubernetes.mdx b/docs/self-hosting/kubernetes.mdx index 33d06f3086c..1939ff5dfb8 100644 --- a/docs/self-hosting/kubernetes.mdx +++ b/docs/self-hosting/kubernetes.mdx @@ -80,6 +80,56 @@ kubectl port-forward svc/trigger-webapp 3040:3030 -n trigger kubectl logs -n trigger deployment/trigger-webapp | grep -A1 "magic link" ``` +## Upgrading + +`helm upgrade` keeps generated secrets, PVCs, and datastore volumes, so most upgrades need no preparation. These chart versions are the exceptions. + +### 4.5.6: two new required secret keys + +The webapp now reads `PROVIDER_SECRET` and `COORDINATOR_SECRET`. When you supply `secrets.existingSecret` the chart generates nothing and reads every key from your Secret, so both keys must exist there before you upgrade. Inline `secrets.*` values are ignored while `existingSecret` is set. + +Add them with two fresh 32-character hex values: + +```bash +kubectl patch secret my-trigger-secrets -n trigger --type merge -p \ + "{\"stringData\":{\"PROVIDER_SECRET\":\"$(openssl rand -hex 16)\",\"COORDINATOR_SECRET\":\"$(openssl rand -hex 16)\"}}" +``` + +The chart checks your Secret in a pre-upgrade hook and aborts with the list of missing keys, leaving the running release untouched. Without that check a missing key surfaces as a `CreateContainerConfigError` partway through the webapp rollout. + +<Note> + GitOps tools that render with `helm template` have no cluster access, so the check is + skipped and a missing key still reaches the rollout. Verify the keys yourself before syncing. +</Note> + +### 4.5.6: ClickHouse credentials moved + +The bundled ClickHouse password moved out of the subchart-generated Secret and into the chart-managed datastore Secret: + +| | Up to 4.5.5 | 4.5.6 and later | +| ------ | ---------------------- | --------------------------- | +| Secret | `<release>-clickhouse` | `trigger-datastore` | +| Key | `admin-password` | `clickhouse-admin-password` | + +The webapp and the ClickHouse server both read the new location, so the upgrade itself needs no action. Repoint anything outside the chart that reads the old Secret — a maintenance CronJob, a Grafana datasource, an external secret sync: + +```bash +kubectl get secret trigger-datastore -n trigger \ + -o jsonpath='{.data.clickhouse-admin-password}' | base64 -d +``` + +The same move applies to the bundled PostgreSQL (`postgres-password`) and MinIO (`minio-root-user`, `minio-root-password`). + +### Bitnami ClickHouse to the official image + +The bundled ClickHouse runs the official `clickhouse/clickhouse-server` image instead of the Bitnami subchart. The chart adopts your existing data volume automatically, with three caveats covered under [ClickHouse](#clickhouse) in external services: rendering without cluster access, pinned Bitnami image tags, and storage that doesn't support `fsGroup`. + +<Note> + The data volume is `ReadWriteOnce`, so during the upgrade the new ClickHouse pod may sit in + `ContainerCreating` with a multi-attach warning for a minute while the old pod still holds the + volume. This clears itself once the old pod is removed later in the same upgrade - don't abort. +</Note> + ## Configuration Most values map directly to the environment variables documented in the [webapp](/self-hosting/env/webapp) and [supervisor](/self-hosting/env/supervisor) environment variable overview. @@ -138,8 +188,9 @@ secrets: # - PROVIDER_SECRET # - COORDINATOR_SECRET # - MANAGED_WORKER_SECRET -# - OBJECT_STORE_ACCESS_KEY_ID -# - OBJECT_STORE_SECRET_ACCESS_KEY +# Plus s3-auth-access-key-id and s3-auth-secret-access-key if you deploy the +# bundled MinIO with s3.auth.existingSecret cleared. The chart lists any keys +# it can't find and fails the install before touching a running release. secrets: enabled: false existingSecret: "your-existing-secret" @@ -279,6 +330,37 @@ redis: #### ClickHouse +<Note>Trigger.dev requires ClickHouse 25.8 or newer.</Note> + +<Note> + When upgrading from a chart version that bundled ClickHouse via the Bitnami subchart, the chart + automatically adopts the existing data volume, so no manual migration is needed. If you render + manifests without cluster access (for example with GitOps tools that use `helm template`), set + `clickhouse.persistence.existingClaim` to the old PVC name + (`data-<release>-clickhouse-shard0-0`) to keep your data — auto-detection can't run there, and + skipping this starts ClickHouse on a fresh empty volume. If that happened, your old data is + still on the old PVC: delete the ClickHouse StatefulSet with `--cascade=orphan` (its volume + configuration is immutable), set `existingClaim`, and sync again. If you render without cluster + access, pin `clickhouse.persistence.existingClaim` for good once the volume is adopted: a later + render where the lookup can't see the PVC would otherwise try to re-add `volumeClaimTemplates`, + which the API server rejects on an existing StatefulSet. +</Note> + +<Note> + If you pinned `clickhouse.image` to a Bitnami repository or tag in your values, update it to + the official `clickhouse/clickhouse-server` image — Bitnami tags don't exist there. The + bundled ClickHouse is single-node: the old Bitnami subchart keys (`shards`, `replicaCount`, + `keeper`) are no longer supported, so use an external ClickHouse for clustered setups. +</Note> + +<Note> + When a volume is adopted from the Bitnami-based chart, a one-time init container fixes its + ownership for the non-root ClickHouse server. This runs automatically on `helm upgrade` (and + when you set `clickhouse.persistence.existingClaim`), so storage that doesn't support `fsGroup` + ownership changes (NFS, hostPath, local-path) works without manual steps. Set + `clickhouse.volumePermissions.enabled: true` to force it in other cases. +</Note> + **Direct configuration:** ```yaml @@ -286,11 +368,18 @@ clickhouse: deploy: false external: host: "my-clickhouse.example.com" - port: 8123 + httpPort: 8123 username: "my-username" password: "my-password" ``` +<Note> + An inline external `username`/`password` is percent-encoded into the connection URL for you, so + store the **raw** value - special characters like `@ : / %` are handled automatically. If you + previously percent-encoded the password by hand to work around this, switch back to the raw value. + (Credentials from `existingSecret` are injected at runtime and are unaffected.) +</Note> + **Using existing secrets (recommended):** ```yaml @@ -298,7 +387,7 @@ clickhouse: deploy: false external: host: "my-clickhouse.example.com" - port: 8123 + httpPort: 8123 username: "my-username" existingSecret: "clickhouse-credentials" # existingSecretKey: "clickhouse-password" # default (optional) diff --git a/hosting/docker/.env.example b/hosting/docker/.env.example index 4c7cf11bc70..b1a797f137d 100644 --- a/hosting/docker/.env.example +++ b/hosting/docker/.env.example @@ -136,7 +136,7 @@ OBJECT_STORE_SECRET_ACCESS_KEY= # POSTGRES_IMAGE_TAG=14 # REDIS_IMAGE_TAG=7 # ELECTRIC_IMAGE_TAG=1.0.13 -# CLICKHOUSE_IMAGE_TAG=latest +# CLICKHOUSE_IMAGE_TAG=26.2 # REGISTRY_IMAGE_TAG=2 # MINIO_IMAGE_TAG=latest # DOCKER_PROXY_IMAGE_TAG=latest diff --git a/hosting/docker/clickhouse/data-paths.xml b/hosting/docker/clickhouse/data-paths.xml new file mode 100644 index 00000000000..bfa9c5ce22a --- /dev/null +++ b/hosting/docker/clickhouse/data-paths.xml @@ -0,0 +1,17 @@ +<!-- + Keeps ClickHouse's on-disk layout compatible with data volumes created by + the previous Bitnami-based setup, which stored everything under a data/ + subdirectory of the volume. Fresh installs get the same layout. tmp lives + outside data/ because old volumes contain a dangling tmp symlink there. +--> +<clickhouse> + <path>/var/lib/clickhouse/data/</path> + <tmp_path>/var/lib/clickhouse/tmp/</tmp_path> + <user_files_path>/var/lib/clickhouse/data/user_files/</user_files_path> + <format_schema_path>/var/lib/clickhouse/data/format_schemas/</format_schema_path> + <user_directories> + <local_directory> + <path>/var/lib/clickhouse/data/access/</path> + </local_directory> + </user_directories> +</clickhouse> diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index 8d133cbeea5..a8bc1167b77 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -80,11 +80,11 @@ services: TRIGGER_BOOTSTRAP_WORKER_GROUP_NAME: bootstrap TRIGGER_BOOTSTRAP_WORKER_TOKEN_PATH: /home/node/shared/worker_token # ClickHouse configuration - CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://default:${CLICKHOUSE_PASSWORD}@clickhouse:8123?secure=false} + CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://${CLICKHOUSE_USER:-default}:${CLICKHOUSE_PASSWORD}@clickhouse:8123?secure=false} CLICKHOUSE_LOG_LEVEL: ${CLICKHOUSE_LOG_LEVEL:-info} # Run replication RUN_REPLICATION_ENABLED: ${RUN_REPLICATION_ENABLED:-1} - RUN_REPLICATION_CLICKHOUSE_URL: ${RUN_REPLICATION_CLICKHOUSE_URL:-http://default:${CLICKHOUSE_PASSWORD}@clickhouse:8123} + RUN_REPLICATION_CLICKHOUSE_URL: ${RUN_REPLICATION_CLICKHOUSE_URL:-http://${CLICKHOUSE_USER:-default}:${CLICKHOUSE_PASSWORD}@clickhouse:8123} RUN_REPLICATION_LOG_LEVEL: ${RUN_REPLICATION_LOG_LEVEL:-info} # Limits # TASK_PAYLOAD_OFFLOAD_THRESHOLD: 524288 # 512KB @@ -157,18 +157,26 @@ services: start_period: 10s clickhouse: - image: bitnamilegacy/clickhouse:${CLICKHOUSE_IMAGE_TAG:-latest} + image: clickhouse/clickhouse-server:${CLICKHOUSE_IMAGE_TAG:-26.2} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config ports: - ${CLICKHOUSE_PUBLISH_IP:-127.0.0.1}:9123:8123 - ${CLICKHOUSE_PUBLISH_IP:-127.0.0.1}:9090:9000 + ulimits: + nofile: + soft: 262144 + hard: 262144 environment: - CLICKHOUSE_ADMIN_USER: ${CLICKHOUSE_USER:-default} - CLICKHOUSE_ADMIN_PASSWORD: ${CLICKHOUSE_PASSWORD:?Set CLICKHOUSE_PASSWORD in .env - run ./generate-secrets.sh} + CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?Set CLICKHOUSE_PASSWORD in .env - run ./generate-secrets.sh} + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 volumes: - - clickhouse:/bitnami/clickhouse - - ../clickhouse/override.xml:/bitnami/clickhouse/etc/config.d/override.xml:ro + # The same volume works across upgrades from the previous Bitnami-based + # setup: data-paths.xml keeps the on-disk layout compatible. + - clickhouse:/var/lib/clickhouse + - ../clickhouse/data-paths.xml:/etc/clickhouse-server/config.d/data-paths.xml:ro + - ../clickhouse/override.xml:/etc/clickhouse-server/config.d/override.xml:ro networks: - webapp healthcheck: diff --git a/hosting/k8s/helm/Chart.lock b/hosting/k8s/helm/Chart.lock index 5a7882cfa0e..338d0db4805 100644 --- a/hosting/k8s/helm/Chart.lock +++ b/hosting/k8s/helm/Chart.lock @@ -5,11 +5,8 @@ dependencies: - name: redis repository: oci://registry-1.docker.io/bitnamicharts version: 21.2.6 -- name: clickhouse - repository: oci://registry-1.docker.io/bitnamicharts - version: 9.4.4 - name: minio repository: oci://registry-1.docker.io/bitnamicharts version: 17.0.9 -digest: sha256:e1b572ab8eca0cc376311398c27b1734d8a598095fccc81dd9c32b2c8b9c1149 -generated: "2026-05-05T10:31:58.493590751+01:00" +digest: sha256:a735954c8b78fcf5b30689bdcdfed66b9be57135368ea696aff2b52ecd731474 +generated: "2026-07-13T16:36:15.800113+01:00" diff --git a/hosting/k8s/helm/Chart.yaml b/hosting/k8s/helm/Chart.yaml index f1753ef7ebc..a1c547e5e64 100644 --- a/hosting/k8s/helm/Chart.yaml +++ b/hosting/k8s/helm/Chart.yaml @@ -26,10 +26,6 @@ dependencies: version: "21.2.6" repository: "oci://registry-1.docker.io/bitnamicharts" condition: redis.deploy - - name: clickhouse - version: "9.4.4" - repository: "oci://registry-1.docker.io/bitnamicharts" - condition: clickhouse.deploy - name: minio version: "17.0.9" repository: "oci://registry-1.docker.io/bitnamicharts" diff --git a/hosting/k8s/helm/templates/_helpers.tpl b/hosting/k8s/helm/templates/_helpers.tpl index 0ecc17b9893..ebd3bf7f021 100644 --- a/hosting/k8s/helm/templates/_helpers.tpl +++ b/hosting/k8s/helm/templates/_helpers.tpl @@ -408,6 +408,38 @@ http://{{ include "trigger-v4.fullname" . }}-s2:{{ .Values.s2.service.port }}/v1 {{- end -}} {{- end }} +{{/* +Percent-encode a string for the userinfo part of a URL. urlquery encodes +spaces as `+` (query semantics), which userinfo decoding keeps literal; a +real `+` becomes `%2B`, so any `+` left in the output is a space and can be +rewritten to `%20`. +*/}} +{{- define "trigger-v4.urlencode" -}} +{{- . | urlquery | replace "+" "%20" -}} +{{- end }} + +{{/* +ClickHouse data-paths config. Keeps the on-disk layout compatible with data +volumes created by the Bitnami subchart this chart used previously, which +stored everything under a data/ subdirectory of the volume. Fresh installs +get the same layout. tmp lives outside data/ because old volumes contain a +dangling tmp symlink there. Users can override by defining their own +data-paths.xml in clickhouse.configdFiles. +*/}} +{{- define "trigger-v4.clickhouse.dataPathsConfig" -}} +<clickhouse> + <path>/var/lib/clickhouse/data/</path> + <tmp_path>/var/lib/clickhouse/tmp/</tmp_path> + <user_files_path>/var/lib/clickhouse/data/user_files/</user_files_path> + <format_schema_path>/var/lib/clickhouse/data/format_schemas/</format_schema_path> + <user_directories> + <local_directory> + <path>/var/lib/clickhouse/data/access/</path> + </local_directory> + </user_directories> +</clickhouse> +{{- end }} + {{/* ClickHouse hostname */}} @@ -415,16 +447,16 @@ ClickHouse hostname {{- if .Values.clickhouse.host }} {{- .Values.clickhouse.host }} {{- else if .Values.clickhouse.deploy }} -{{- printf "%s-clickhouse" .Release.Name }} +{{- printf "%s-clickhouse" (include "trigger-v4.fullname" .) }} {{- end }} {{- end }} {{/* ClickHouse URL for application (with secure parameter) -Note on the external+existingSecret branch: the password is expanded via -Kubernetes' `$(VAR)` syntax, not shell `${VAR}`. Kubelet substitutes -`$(CLICKHOUSE_PASSWORD)` at container-creation time from the +Note on the deploy and external+existingSecret branches: the password is +expanded via Kubernetes' `$(VAR)` syntax, not shell `${VAR}`. Kubelet +substitutes `$(CLICKHOUSE_PASSWORD)` at container-creation time from the CLICKHOUSE_PASSWORD env var declared just before CLICKHOUSE_URL in webapp.yaml. Shell-style `${...}` does not work here because `docker/scripts/entrypoint.sh` assigns CLICKHOUSE_URL to GOOSE_DBSTRING @@ -432,21 +464,25 @@ with a single-pass expansion (`export GOOSE_DBSTRING="$CLICKHOUSE_URL"`), so any inner `${...}` reaches goose verbatim and fails URL parsing. CLICKHOUSE_PASSWORD must contain only URL-userinfo-safe characters — the -value is substituted verbatim, so `@ : / ? # [ ] %` break the URL. Use a -hex-encoded password or percent-encode before storing in the Secret. +value is substituted verbatim, so `@ : / ? # [ ] %` break the URL. The +chart-generated datastore password is hex, which is safe; a pinned +auth.password or external Secret value must be URL-safe too. + +Inline credentials (usernames and the external plain password) are +percent-encoded, so any special characters are safe there. */}} {{- define "trigger-v4.clickhouse.url" -}} {{- if .Values.clickhouse.deploy -}} {{- $protocol := ternary "https" "http" .Values.clickhouse.secure -}} {{- $secure := ternary "true" "false" .Values.clickhouse.secure -}} -{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:8123?secure={{ $secure }} +{{ $protocol }}://{{ include "trigger-v4.urlencode" .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:{{ .Values.clickhouse.service.ports.http }}?secure={{ $secure }} {{- else if .Values.clickhouse.external.host -}} {{- $protocol := ternary "https" "http" .Values.clickhouse.external.secure -}} {{- $secure := ternary "true" "false" .Values.clickhouse.external.secure -}} {{- if .Values.clickhouse.external.existingSecret -}} -{{ $protocol }}://{{ .Values.clickhouse.external.username }}:$(CLICKHOUSE_PASSWORD)@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}?secure={{ $secure }} +{{ $protocol }}://{{ include "trigger-v4.urlencode" .Values.clickhouse.external.username }}:$(CLICKHOUSE_PASSWORD)@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}?secure={{ $secure }} {{- else -}} -{{ $protocol }}://{{ .Values.clickhouse.external.username }}:{{ .Values.clickhouse.external.password }}@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}?secure={{ $secure }} +{{ $protocol }}://{{ include "trigger-v4.urlencode" .Values.clickhouse.external.username }}:{{ include "trigger-v4.urlencode" .Values.clickhouse.external.password }}@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }}?secure={{ $secure }} {{- end -}} {{- end -}} {{- end }} @@ -460,13 +496,13 @@ applies to the replication URL. {{- define "trigger-v4.clickhouse.replication.url" -}} {{- if .Values.clickhouse.deploy -}} {{- $protocol := ternary "https" "http" .Values.clickhouse.secure -}} -{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:8123 +{{ $protocol }}://{{ include "trigger-v4.urlencode" .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:{{ .Values.clickhouse.service.ports.http }} {{- else if .Values.clickhouse.external.host -}} {{- $protocol := ternary "https" "http" .Values.clickhouse.external.secure -}} {{- if .Values.clickhouse.external.existingSecret -}} -{{ $protocol }}://{{ .Values.clickhouse.external.username }}:$(CLICKHOUSE_PASSWORD)@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }} +{{ $protocol }}://{{ include "trigger-v4.urlencode" .Values.clickhouse.external.username }}:$(CLICKHOUSE_PASSWORD)@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }} {{- else -}} -{{ $protocol }}://{{ .Values.clickhouse.external.username }}:{{ .Values.clickhouse.external.password }}@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }} +{{ $protocol }}://{{ include "trigger-v4.urlencode" .Values.clickhouse.external.username }}:{{ include "trigger-v4.urlencode" .Values.clickhouse.external.password }}@{{ .Values.clickhouse.external.host }}:{{ .Values.clickhouse.external.httpPort | default 8123 }} {{- end -}} {{- end -}} {{- end }} diff --git a/hosting/k8s/helm/templates/clickhouse.yaml b/hosting/k8s/helm/templates/clickhouse.yaml new file mode 100644 index 00000000000..c4baca8d4da --- /dev/null +++ b/hosting/k8s/helm/templates/clickhouse.yaml @@ -0,0 +1,242 @@ +{{- if .Values.clickhouse.deploy }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "trigger-v4.fullname" . }}-clickhouse-config + labels: + {{- $component := "clickhouse" }} + {{- include "trigger-v4.componentLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }} +data: + {{- if not (hasKey .Values.clickhouse.configdFiles "data-paths.xml") }} + data-paths.xml: | + {{- include "trigger-v4.clickhouse.dataPathsConfig" . | nindent 4 }} + {{- end }} + {{- range $filename, $content := .Values.clickhouse.configdFiles }} + {{ $filename }}: | + {{- $content | nindent 4 }} + {{- end }} +--- +{{- /* Reuse an existing data PVC instead of creating one via + volumeClaimTemplates. Set explicitly through persistence.existingClaim, + or detected automatically: upgrades from chart versions that bundled + the Bitnami subchart leave their PVC behind under the old name, and + adopting it preserves all ClickHouse data with no manual migration. + (lookup returns nothing during template/dry-run rendering; set + persistence.existingClaim explicitly when pre-rendering manifests, + e.g. with GitOps tools.) */}} +{{- $existingClaim := .Values.clickhouse.persistence.existingClaim }} +{{- if and (not $existingClaim) .Values.clickhouse.persistence.enabled }} +{{- $legacyName := printf "data-%s-clickhouse-shard0-0" .Release.Name }} +{{- if lookup "v1" "PersistentVolumeClaim" .Release.Namespace $legacyName }} +{{- $existingClaim = $legacyName }} +{{- end }} +{{- end }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "trigger-v4.fullname" . }}-clickhouse + labels: + {{- $component := "clickhouse" }} + {{- include "trigger-v4.componentLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }} +spec: + replicas: 1 + serviceName: {{ include "trigger-v4.fullname" . }}-clickhouse + selector: + matchLabels: + {{- include "trigger-v4.componentSelectorLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ printf "%s\n%s" (include "trigger-v4.clickhouse.dataPathsConfig" .) (.Values.clickhouse.configdFiles | toYaml) | sha256sum }} + {{- /* Restart on pinned-password changes; when the password is + auto-generated it lives in the retained datastore secret and + never rotates on upgrade, so there is nothing to hash. */}} + {{- if .Values.clickhouse.auth.password }} + checksum/secret: {{ .Values.clickhouse.auth.password | sha256sum }} + {{- end }} + {{- with .Values.clickhouse.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "trigger-v4.componentSelectorLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 8 }} + spec: + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.clickhouse.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- /* Adopting a volume from the Bitnami-based chart (auto-detected or an + explicit existingClaim) means the data is owned by the old uid. On + storage where fsGroup can't relabel it (hostPath, NFS, local-path), + the non-root server can't read it, so run the chown init container + automatically. Fresh installs create their own data and skip it. */}} + {{- if or .Values.clickhouse.volumePermissions.enabled $existingClaim }} + initContainers: + - name: volume-permissions + image: "{{ .Values.global.imageRegistry | default .Values.clickhouse.volumePermissions.image.registry }}/{{ .Values.clickhouse.volumePermissions.image.repository }}:{{ .Values.clickhouse.volumePermissions.image.tag }}" + imagePullPolicy: {{ .Values.clickhouse.volumePermissions.image.pullPolicy }} + {{- $chownUser := .Values.clickhouse.securityContext.runAsUser | default 101 }} + {{- $chownGroup := .Values.clickhouse.securityContext.runAsGroup | default 101 }} + {{- /* Idempotent: skip the recursive chown when the volume root is already + owned by the run-as user, so it doesn't re-walk the whole database on + every restart (only the first mount after adoption pays the cost). */}} + command: ["sh", "-c", "test \"$(stat -c %u /var/lib/clickhouse)\" = \"{{ $chownUser }}\" || chown -R {{ $chownUser }}:{{ $chownGroup }} /var/lib/clickhouse"] + securityContext: + runAsUser: 0 + runAsNonRoot: false + volumeMounts: + - name: data + mountPath: /var/lib/clickhouse + {{- end }} + containers: + - name: clickhouse + {{- with .Values.clickhouse.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: "{{ .Values.global.imageRegistry | default .Values.clickhouse.image.registry }}/{{ .Values.clickhouse.image.repository }}:{{ .Values.clickhouse.image.tag }}{{ with .Values.clickhouse.image.digest }}@{{ . }}{{ end }}" + imagePullPolicy: {{ .Values.clickhouse.image.pullPolicy }} + env: + - name: CLICKHOUSE_USER + value: {{ .Values.clickhouse.auth.username | quote }} + {{- /* Same chart-managed datastore secret the webapp reads for its + connection URL, so the server credential and the app's URL + always match. */}} + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.clickhouse.auth.existingSecret | default (include "trigger-v4.datastore.secretName" .) }} + key: {{ .Values.clickhouse.auth.existingSecretKey | default "clickhouse-admin-password" }} + - name: CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT + value: "1" + ports: + - name: http + containerPort: 8123 + protocol: TCP + - name: native + containerPort: 9000 + protocol: TCP + {{- if .Values.clickhouse.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: {{ .Values.clickhouse.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.clickhouse.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.clickhouse.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.clickhouse.livenessProbe.failureThreshold }} + successThreshold: {{ .Values.clickhouse.livenessProbe.successThreshold }} + {{- end }} + {{- if .Values.clickhouse.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: {{ .Values.clickhouse.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.clickhouse.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.clickhouse.readinessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.clickhouse.readinessProbe.failureThreshold }} + successThreshold: {{ .Values.clickhouse.readinessProbe.successThreshold }} + {{- end }} + {{- if .Values.clickhouse.startupProbe.enabled }} + startupProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: {{ .Values.clickhouse.startupProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.clickhouse.startupProbe.periodSeconds }} + timeoutSeconds: {{ .Values.clickhouse.startupProbe.timeoutSeconds }} + failureThreshold: {{ .Values.clickhouse.startupProbe.failureThreshold }} + successThreshold: {{ .Values.clickhouse.startupProbe.successThreshold }} + {{- end }} + resources: + {{- toYaml .Values.clickhouse.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: /var/lib/clickhouse + - name: logs + mountPath: /var/log/clickhouse-server + {{- /* Mount each override file individually: shadowing the whole + config.d directory would remove the image's built-in + docker_related_config.xml, which makes the server listen on + 0.0.0.0 instead of localhost only. */}} + {{- if not (hasKey .Values.clickhouse.configdFiles "data-paths.xml") }} + - name: config + mountPath: /etc/clickhouse-server/config.d/data-paths.xml + subPath: data-paths.xml + {{- end }} + {{- range $filename, $_ := .Values.clickhouse.configdFiles }} + - name: config + mountPath: /etc/clickhouse-server/config.d/{{ $filename }} + subPath: {{ $filename }} + {{- end }} + {{- with .Values.clickhouse.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.clickhouse.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.clickhouse.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ include "trigger-v4.fullname" . }}-clickhouse-config + - name: logs + emptyDir: {} + {{- if not .Values.clickhouse.persistence.enabled }} + - name: data + emptyDir: {} + {{- else if $existingClaim }} + - name: data + persistentVolumeClaim: + claimName: {{ $existingClaim }} + {{- end }} + {{- if and .Values.clickhouse.persistence.enabled (not $existingClaim) }} + volumeClaimTemplates: + - metadata: + name: data + {{- if .Values.clickhouse.persistence.retain }} + annotations: + helm.sh/resource-policy: keep + {{- end }} + spec: + accessModes: + - {{ .Values.clickhouse.persistence.accessMode }} + resources: + requests: + storage: {{ .Values.clickhouse.persistence.size }} + {{- $storageClass := .Values.clickhouse.persistence.storageClass | default .Values.global.storageClass }} + {{- if $storageClass }} + storageClassName: {{ $storageClass }} + {{- end }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "trigger-v4.fullname" . }}-clickhouse + labels: + {{- $component := "clickhouse" }} + {{- include "trigger-v4.componentLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }} +spec: + type: {{ .Values.clickhouse.service.type }} + ports: + - port: {{ .Values.clickhouse.service.ports.http }} + targetPort: http + protocol: TCP + name: http + - port: {{ .Values.clickhouse.service.ports.native }} + targetPort: native + protocol: TCP + name: native + selector: + {{- include "trigger-v4.componentSelectorLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }} +{{- end }} diff --git a/hosting/k8s/helm/templates/tests/test-clickhouse.yaml b/hosting/k8s/helm/templates/tests/test-clickhouse.yaml index 9bde62c2ada..bb71e381b71 100644 --- a/hosting/k8s/helm/templates/tests/test-clickhouse.yaml +++ b/hosting/k8s/helm/templates/tests/test-clickhouse.yaml @@ -12,10 +12,18 @@ spec: containers: - name: test-clickhouse image: curlimages/curl:8.14.1 + env: + - name: CLICKHOUSE_USER + value: {{ .Values.clickhouse.auth.username | quote }} + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.clickhouse.auth.existingSecret | default (include "trigger-v4.datastore.secretName" .) }} + key: {{ .Values.clickhouse.auth.existingSecretKey | default "clickhouse-admin-password" }} command: ['sh', '-c'] args: - | echo "Testing ClickHouse HTTP interface..." - curl -f --user "{{ .Values.clickhouse.auth.adminUser }}:{{ .Values.clickhouse.auth.adminPassword }}" "http://{{ include "trigger-v4.fullname" . }}-clickhouse:{{ .Values.clickhouse.service.ports.http }}/ping" + curl -f --user "$CLICKHOUSE_USER:$CLICKHOUSE_PASSWORD" "http://{{ include "trigger-v4.clickhouse.hostname" . }}:{{ .Values.clickhouse.service.ports.http }}/ping" echo "ClickHouse test completed successfully" -{{- end }} \ No newline at end of file +{{- end }} diff --git a/hosting/k8s/helm/templates/validate-external-config.yaml b/hosting/k8s/helm/templates/validate-external-config.yaml index 939e699a423..7c5b25087aa 100644 --- a/hosting/k8s/helm/templates/validate-external-config.yaml +++ b/hosting/k8s/helm/templates/validate-external-config.yaml @@ -54,7 +54,36 @@ Application and control-plane secrets are auto-generated by templates/secrets.ya when left unset (retained across upgrades via lookup), so they need no fail-closed guard here. The webapp still rejects previously published values at startup, even when supplied via secrets.existingSecret. + +With secrets.existingSecret the chart generates nothing, so every key the workloads +reference has to already be present in that Secret. Report missing keys up front - +otherwise a chart version that starts consuming a new key only surfaces it as a +CreateContainerConfigError partway through the rollout. The check is skipped when +`lookup` returns nothing (helm template, client-side dry-run, or a Secret created +later in the same apply): it can only report on what it can read. */}} +{{- if .Values.secrets.existingSecret }} +{{- $required := list "SESSION_SECRET" "MAGIC_LINK_SECRET" "ENCRYPTION_KEY" "PROVIDER_SECRET" "COORDINATOR_SECRET" "MANAGED_WORKER_SECRET" }} +{{- if and .Values.s3.deploy (not .Values.s3.auth.existingSecret) }} +{{- $required = concat $required (list "s3-auth-access-key-id" "s3-auth-secret-access-key") }} +{{- end }} +{{- if and (not .Values.s3.deploy) (not .Values.s3.external.existingSecret) .Values.s3.external.accessKeyId }} +{{- $required = concat $required (list "s3-access-key-id" "s3-secret-access-key") }} +{{- end }} +{{- $found := lookup "v1" "Secret" .Release.Namespace .Values.secrets.existingSecret }} +{{- if $found }} +{{- $data := (get $found "data") | default dict }} +{{- $missing := list }} +{{- range $key := $required }} +{{- if not (hasKey $data $key) }} +{{- $missing = append $missing $key }} +{{- end }} +{{- end }} +{{- if $missing }} +{{- fail (printf "Secret %q (secrets.existingSecret) is missing required keys: %s. Add them before upgrading - while secrets.existingSecret is set the chart generates nothing and inline secrets.* values are ignored. See https://trigger.dev/docs/self-hosting/kubernetes#upgrading" .Values.secrets.existingSecret (join ", " $missing)) }} +{{- end }} +{{- end }} +{{- end }} {{/* This template produces no output but will fail the deployment if validation fails diff --git a/hosting/k8s/helm/templates/webapp.yaml b/hosting/k8s/helm/templates/webapp.yaml index 49acee98f4a..d666237daaf 100644 --- a/hosting/k8s/helm/templates/webapp.yaml +++ b/hosting/k8s/helm/templates/webapp.yaml @@ -419,11 +419,13 @@ spec: name: {{ include "trigger-v4.clickhouse.external.secretName" . }} key: {{ include "trigger-v4.clickhouse.external.passwordKey" . }} {{- else if .Values.clickhouse.deploy }} + {{- /* Same secret reference the bundled ClickHouse server uses, so a + custom auth.existingSecret keeps the app and server in sync. */}} - name: CLICKHOUSE_PASSWORD valueFrom: secretKeyRef: - name: {{ include "trigger-v4.datastore.secretName" . }} - key: clickhouse-admin-password + name: {{ .Values.clickhouse.auth.existingSecret | default (include "trigger-v4.datastore.secretName" .) }} + key: {{ .Values.clickhouse.auth.existingSecretKey | default "clickhouse-admin-password" }} {{- end }} - name: CLICKHOUSE_URL value: {{ include "trigger-v4.clickhouse.url" . | quote }} diff --git a/hosting/k8s/helm/values-production-example.yaml b/hosting/k8s/helm/values-production-example.yaml index ee99de58a9b..3f99547fcf0 100644 --- a/hosting/k8s/helm/values-production-example.yaml +++ b/hosting/k8s/helm/values-production-example.yaml @@ -77,12 +77,12 @@ redis: memory: 512Mi # Production ClickHouse +# The bundled ClickHouse serves plain HTTP inside the cluster. For TLS, +# use an external ClickHouse (deploy: false) with secure: true (see below). clickhouse: auth: # Required — no built-in default. The webapp connection string uses clickhouse.auth.password. password: "your-strong-clickhouse-password" - # Set to true to enable TLS/secure connections in production - secure: true persistence: enabled: true size: 100Gi diff --git a/hosting/k8s/helm/values.yaml b/hosting/k8s/helm/values.yaml index e76a921279c..e6ea51864f1 100644 --- a/hosting/k8s/helm/values.yaml +++ b/hosting/k8s/helm/values.yaml @@ -628,19 +628,24 @@ s2: existingSecretAccessTokenKey: "access-token" # ClickHouse configuration -# Subchart: https://github.com/bitnami/charts/tree/main/bitnami/clickhouse +# Deploys a single-node ClickHouse using the official image: +# https://hub.docker.com/r/clickhouse/clickhouse-server +# For clustered/replicated setups, use an external ClickHouse (deploy: false). clickhouse: deploy: true image: - # Use bitnami legacy repo - repository: bitnamilegacy/clickhouse - # image: docker.io/bitnamilegacy/clickhouse:25.7.5-debian-12-r0 + registry: docker.io + repository: clickhouse/clickhouse-server + # Trigger.dev requires ClickHouse >= 25.8 + tag: "26.2" + # Pinning by digest is strongly recommended for reproducible deployments + digest: "" + pullPolicy: IfNotPresent # TLS/Secure connection configuration secure: false # Set to true to use HTTPS and secure connections - # Bitnami ClickHouse chart configuration (when deploy: true) auth: username: "default" password: "" # Leave empty to auto-generate into the datastore secret, or set to pin. @@ -648,23 +653,92 @@ clickhouse: existingSecret: "trigger-datastore" existingSecretKey: "clickhouse-admin-password" - # Single-node configuration (disable clustering for dev/test) - keeper: + podAnnotations: {} + + # The official image runs ClickHouse as uid 101. fsGroup makes the persistent + # volume writable by that user without running the container as root, and + # OnRootMismatch relabels volumes carried over from older chart versions + # (different uid) on first mount without rechecking every file on later mounts. + podSecurityContext: + fsGroup: 101 + fsGroupChangePolicy: OnRootMismatch + securityContext: + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 101 + + # One-time root init container that chowns the data volume to the ClickHouse + # uid. Only needed on storage that doesn't support fsGroup ownership changes + # (e.g. NFS, hostPath); on such storage a data volume carried over from the + # Bitnami-based chart is otherwise unreadable by the non-root server. + volumePermissions: enabled: false + image: + registry: docker.io + repository: busybox + tag: "1.35" + pullPolicy: IfNotPresent - shards: 1 - replicaCount: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + + service: + type: ClusterIP + ports: + http: 8123 + native: 9000 persistence: enabled: true size: 10Gi + accessMode: ReadWriteOnce + storageClass: "" + retain: false + # Name of an existing PVC to use for ClickHouse data instead of creating + # one. Normally left empty: upgrades from chart versions that bundled the + # Bitnami ClickHouse subchart adopt the old data PVC automatically. Set + # this explicitly when rendering manifests without cluster access (e.g. + # GitOps tools that use `helm template`), where auto-detection can't run: + # the old PVC is named data-<release>-clickhouse-shard0-0. + existingClaim: "" ## ClickHouse resource requests and limits ## ref: http://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ - ## @param resourcesPreset Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production). - ## More information: https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15 - resourcesPreset: "xlarge" - resources: {} + ## ClickHouse can be very resource intensive. The defaults below match the + ## resource preset the chart previously applied; size them to your workload + ## for production (see values-production-example.yaml). + resources: + requests: + cpu: 1000m + memory: 3Gi + limits: + cpu: 3000m + memory: 6Gi + + livenessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 5 + successThreshold: 1 + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 5 + successThreshold: 1 + # Generous startup window: first boot on a large adopted data volume can + # spend a while loading metadata before the HTTP listener answers. + startupProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 60 + successThreshold: 1 # External ClickHouse connection (when deploy: false) external: From 14824b09556ac3310643174ec33a405d909bb129 Mon Sep 17 00:00:00 2001 From: James Ritchie <james@trigger.dev> Date: Sat, 1 Aug 2026 16:26:26 +0100 Subject: [PATCH 56/67] feat(webapp): fix agent overview page scroll bug + layout fixes on task and agent pages (#4454) ## Summary The task, scheduled task and agent pages now name their runs table with its own title bar, and the controls that page the table sit beside it rather than in the bar at the top of the page. The top bar keeps just the date filter. Two agent page layout bugs are fixed along the way: scrolling a wide runs table sideways dragged the charts off screen with it, and the details panel stopped short of the bottom of the window. ## Fix The charts moved because the runs table had no horizontal scroller of its own. `stickyHeader` swaps the table's `overflow-x-auto` for `overflow-visible`, so the overflow escaped up to the page scroll box, and setting only `overflow-y-auto` on that box leaves the computed `overflow-x` at `visible`, which CSS then promotes to `auto`. The chart grid is a sibling inside that box, so it scrolled too. The table now keeps its own scroller (the same rule the queues list already documents) and the page box clips x so this cannot recur. The short panel was a second `PageContainer` wrapping the agent routes. `PageContainer` is `grid-rows-[auto_1fr]`, so a lone child lands in the `auto` row and its `h-full` resolves against content height instead of the viewport. This also reverts the global tooltip `max-w-[230px]` introduced in [#4131](https://github.com/triggerdotdev/trigger.dev/pull/4131), so longer tooltips are no longer squeezed into a narrow column. ### Agent overview page showing table now scrolling <img width="3452" height="1648" alt="CleanShot 2026-08-01 at 12 04 38@2x" src="https://github.com/user-attachments/assets/ef1ac55d-8ffb-4278-983b-031ed21c1f55" /> --- .../app/components/layout/MetricsLayout.tsx | 6 +- .../app/components/primitives/Headers.tsx | 2 +- .../app/components/primitives/Table.tsx | 2 +- .../webapp/app/components/primitives/Tabs.tsx | 79 ++++++++++++++-- .../app/components/primitives/TitleBar.tsx | 26 ++++++ .../app/components/primitives/Tooltip.tsx | 2 +- .../route.tsx | 81 ++++++++-------- .../route.tsx | 8 +- .../route.tsx | 92 ++++++++++--------- .../route.tsx | 68 +++++++------- 10 files changed, 236 insertions(+), 130 deletions(-) create mode 100644 apps/webapp/app/components/primitives/TitleBar.tsx diff --git a/apps/webapp/app/components/layout/MetricsLayout.tsx b/apps/webapp/app/components/layout/MetricsLayout.tsx index 12d0fcd743c..9a41c4b2613 100644 --- a/apps/webapp/app/components/layout/MetricsLayout.tsx +++ b/apps/webapp/app/components/layout/MetricsLayout.tsx @@ -175,10 +175,12 @@ function MetricsLayoutMain({ children, scroll }: { children: ReactNode; scroll: return ( <div className="flex h-full min-h-0 flex-col"> {filters} + {/* overflow-x-clip: without it `overflow-y-auto` promotes x to auto and wide content drags + the charts sideways. Wide children must scroll in their own container. */} <div className={ scroll === "page" - ? "flex min-h-0 flex-1 flex-col gap-2.5 overflow-y-auto py-2.5 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-surface-control" + ? "flex min-h-0 flex-1 flex-col gap-2.5 overflow-y-auto overflow-x-clip py-2.5 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-surface-control" : "flex min-h-0 flex-1 flex-col overflow-hidden" } > @@ -286,7 +288,7 @@ function MetricsLayoutFilters({ return ( <div className={cn( - "flex h-10 shrink-0 items-center justify-between gap-2 border-b border-grid-dimmed pl-2.5 pr-3", + "flex h-10 shrink-0 items-center justify-between gap-2 border-b border-grid-dimmed px-2", className )} > diff --git a/apps/webapp/app/components/primitives/Headers.tsx b/apps/webapp/app/components/primitives/Headers.tsx index 53c9d822994..5cd3ec84559 100644 --- a/apps/webapp/app/components/primitives/Headers.tsx +++ b/apps/webapp/app/components/primitives/Headers.tsx @@ -1,6 +1,6 @@ import { cn } from "~/utils/cn"; -const headerVariants = { +export const headerVariants = { header1: { text: "font-sans text-2xl leading-5 md:leading-6 lg:leading-7 font-semibold tracking-tight", spacing: "mb-2", diff --git a/apps/webapp/app/components/primitives/Table.tsx b/apps/webapp/app/components/primitives/Table.tsx index dccb26df454..55dd2e4e200 100644 --- a/apps/webapp/app/components/primitives/Table.tsx +++ b/apps/webapp/app/components/primitives/Table.tsx @@ -181,7 +181,7 @@ type TableCellBasicProps = { type TableHeaderCellProps = TableCellBasicProps & { hiddenLabel?: boolean; tooltip?: ReactNode; - /** Extra class merged onto the tooltip content — e.g. widen it past the default max-width. */ + /** Extra class merged onto the tooltip content. */ tooltipContentClassName?: string; disableTooltipHoverableContent?: boolean; /** diff --git a/apps/webapp/app/components/primitives/Tabs.tsx b/apps/webapp/app/components/primitives/Tabs.tsx index 1128a9c267f..569e271434f 100644 --- a/apps/webapp/app/components/primitives/Tabs.tsx +++ b/apps/webapp/app/components/primitives/Tabs.tsx @@ -3,9 +3,19 @@ import { motion } from "framer-motion"; import { type ReactNode, useRef } from "react"; import { type ShortcutDefinition, useShortcutKeys } from "~/hooks/useShortcutKeys"; import { cn } from "~/utils/cn"; +import { headerVariants } from "./Headers"; import { ShortcutKey } from "./ShortcutKey"; -export type Variants = "underline" | "pipe-divider" | "segmented"; +/** `"title"` names the table below it: header2 text, filter-bar height, underline on the border. */ +export type Variants = "underline" | "pipe-divider" | "segmented" | "title"; + +/** Shared with `TitleBar` so the tabbed and tab-less bars match. */ +export const TITLE_BAR_CHROME = "flex h-10 shrink-0 gap-x-6 border-b border-grid-bright"; + +const titleTabLabel = cn(headerVariants.header2.text, "transition duration-200"); +const titleTabIndicator = "h-0.5 w-full bg-indigo-500"; +const titleTabIndicatorIdle = + "h-0.5 w-full bg-surface-control-active opacity-0 transition duration-200 group-hover:opacity-100"; export type TabsProps = { tabs: { @@ -58,6 +68,10 @@ export function TabContainer({ ); } + if (variant === "title") { + return <div className={cn(TITLE_BAR_CHROME, "items-stretch", className)}>{children}</div>; + } + if (variant === "underline") { return ( <div className={cn(`flex gap-x-6 border-b border-grid-bright`, className)}>{children}</div> @@ -117,6 +131,39 @@ export function TabLink({ ); } + if (variant === "title") { + return ( + <NavLink to={to} className="group flex h-full flex-col focus-custom" end={end}> + {({ isActive, isPending }) => { + const active = isActive || isPending; + return ( + <> + <div className="flex flex-1 items-center"> + <span + className={cn( + titleTabLabel, + active ? "text-text-bright" : "text-text-dimmed group-hover:text-text-bright" + )} + > + {children} + </span> + </div> + {active ? ( + <motion.div + layoutId={layoutId} + transition={{ type: "spring", stiffness: 500, damping: 30 }} + className={titleTabIndicator} + /> + ) : ( + <div className={titleTabIndicatorIdle} /> + )} + </> + ); + }} + </NavLink> + ); + } + if (variant === "pipe-divider") { return ( <NavLink @@ -177,11 +224,13 @@ export function TabButton({ isActive, layoutId, shortcut, + variant = "underline", ...props }: { isActive: boolean; shortcut?: ShortcutDefinition; layoutId: string; + variant?: Variants; } & React.ButtonHTMLAttributes<HTMLButtonElement>) { const ref = useRef<HTMLButtonElement>(null); @@ -197,10 +246,13 @@ export function TabButton({ }); } + const title = variant === "title"; + return ( <button className={cn( - "group flex flex-col items-center pt-1 focus-custom", + "group flex flex-col items-center focus-custom", + title ? "h-full" : "pt-1", props.className, props.disabled && "pointer-events-none opacity-50" )} @@ -209,8 +261,18 @@ export function TabButton({ {...props} > <> - <div className="flex items-center gap-1"> - <span className={"text-sm transition duration-200 text-text-bright"}> + <div className={cn("flex items-center gap-1", title && "flex-1")}> + <span + className={cn( + "transition duration-200", + title + ? cn( + headerVariants.header2.text, + isActive ? "text-text-bright" : "text-text-dimmed group-hover:text-text-bright" + ) + : "text-sm text-text-bright" + )} + > {props.children} </span> {shortcut && <ShortcutKey className={cn("")} shortcut={shortcut} variant={"small"} />} @@ -219,10 +281,15 @@ export function TabButton({ <motion.div layoutId={layoutId} transition={{ type: "spring", stiffness: 500, damping: 30 }} - className="mt-1 h-0.5 w-full bg-indigo-500" + className={cn("h-0.5 w-full bg-indigo-500", !title && "mt-1")} /> ) : ( - <div className="mt-1 h-0.5 w-full bg-surface-control-active opacity-0 transition duration-200 group-hover:opacity-100" /> + <div + className={cn( + "h-0.5 w-full bg-surface-control-active opacity-0 transition duration-200 group-hover:opacity-100", + !title && "mt-1" + )} + /> )} </> </button> diff --git a/apps/webapp/app/components/primitives/TitleBar.tsx b/apps/webapp/app/components/primitives/TitleBar.tsx new file mode 100644 index 00000000000..8b935093b02 --- /dev/null +++ b/apps/webapp/app/components/primitives/TitleBar.tsx @@ -0,0 +1,26 @@ +import { type ReactNode } from "react"; +import { cn } from "~/utils/cn"; +import { Header2 } from "./Headers"; +import { TITLE_BAR_CHROME } from "./Tabs"; + +/** + * Names the table below it. Bottom rule only — it doubles as the table's top edge, so render the + * table with `showTopBorder={false}`. Use `TabContainer variant="title"` for the tabbed form. + */ +export function TitleBar({ + title, + children, + className, +}: { + title: ReactNode; + /** Right-aligned controls. */ + children?: ReactNode; + className?: string; +}) { + return ( + <div className={cn(TITLE_BAR_CHROME, "items-center justify-between pl-2.5 pr-1.5", className)}> + <Header2>{title}</Header2> + {children ? <div className="flex items-center gap-1.5">{children}</div> : null} + </div> + ); +} diff --git a/apps/webapp/app/components/primitives/Tooltip.tsx b/apps/webapp/app/components/primitives/Tooltip.tsx index 6f12e408c7f..cb9eaf0364d 100644 --- a/apps/webapp/app/components/primitives/Tooltip.tsx +++ b/apps/webapp/app/components/primitives/Tooltip.tsx @@ -42,7 +42,7 @@ const TooltipContent = React.forwardRef< ref={ref} sideOffset={sideOffset} className={cn( - "z-50 max-w-[230px] overflow-hidden animate-in data-[side=bottom]:slide-in-from-top-1 data-[side=left]:slide-in-from-right-1 data-[side=right]:slide-in-from-left-1 data-[side=top]:slide-in-from-bottom-1 focus-visible:outline-hidden", + "z-50 overflow-hidden animate-in data-[side=bottom]:slide-in-from-top-1 data-[side=left]:slide-in-from-right-1 data-[side=right]:slide-in-from-left-1 data-[side=top]:slide-in-from-bottom-1 focus-visible:outline-hidden", variantClasses[variant], className )} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx index 76beaf297f3..5d9a86e4a25 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx @@ -234,28 +234,10 @@ export default function Page() { </PageAccessories> </NavBar> <MetricsLayout.Root> - {/* Filters — the pinned bar under the NavBar: the TimeFilter and pagination that used to - be fused with the tabs now live here, above the charts (Queues list pattern). Left and - right clusters are child divs; the slot's baked justify-between spreads them. */} <MetricsLayout.Filters> <div className="flex items-center gap-2"> <TimeFilter defaultPeriod="7d" labelName={tabLabel} /> </div> - <div className="flex items-center gap-2"> - {tab === "sessions" ? ( - <Suspense fallback={null}> - <TypedAwait resolve={sessionList} errorElement={null}> - {(list) => (list ? <ListPagination list={list} /> : null)} - </TypedAwait> - </Suspense> - ) : ( - <Suspense fallback={null}> - <TypedAwait resolve={runList} errorElement={null}> - {(list) => (list ? <ListPagination list={list} /> : null)} - </TypedAwait> - </Suspense> - )} - </div> </MetricsLayout.Filters> {/* Activity / LLM spend / Token charts as a fixed-height chart row (three-up), synced + @@ -315,23 +297,45 @@ export default function Page() { {/* Tabs alone on their row (Queue detail pattern), then the table below them. */} <MetricsLayout.Content> - <TabContainer className="px-3"> - <TabButton - isActive={tab === "sessions"} - layoutId="agent-page-tabs" - onClick={() => setTab("sessions")} - > - Sessions - </TabButton> - <TabButton - isActive={tab === "runs"} - layoutId="agent-page-tabs" - onClick={() => setTab("runs")} - > - Runs - </TabButton> - </TabContainer> - <AgentContentArea tab={tab} sessionList={sessionList} runList={runList} /> + {/* Single child so Content's gap-2.5 can't separate the bar from the table. */} + <div className="flex flex-col"> + <TabContainer variant="title" className="justify-between border-y px-2"> + <div className="flex items-stretch gap-x-6"> + <TabButton + isActive={tab === "sessions"} + layoutId="agent-page-tabs" + variant="title" + onClick={() => setTab("sessions")} + > + Sessions + </TabButton> + <TabButton + isActive={tab === "runs"} + layoutId="agent-page-tabs" + variant="title" + onClick={() => setTab("runs")} + > + Runs + </TabButton> + </div> + <div className="flex items-center gap-1.5"> + {tab === "sessions" ? ( + <Suspense fallback={null}> + <TypedAwait resolve={sessionList} errorElement={null}> + {(list) => (list ? <ListPagination list={list} /> : null)} + </TypedAwait> + </Suspense> + ) : ( + <Suspense fallback={null}> + <TypedAwait resolve={runList} errorElement={null}> + {(list) => (list ? <ListPagination list={list} /> : null)} + </TypedAwait> + </Suspense> + )} + </div> + </TabContainer> + <AgentContentArea tab={tab} sessionList={sessionList} runList={runList} /> + </div> </MetricsLayout.Content> <MetricsLayout.Sidebar @@ -356,8 +360,7 @@ function AgentContentArea({ sessionList, runList, }: { tab: AgentTab } & Pick<LoaderData, "sessionList" | "runList">) { - // The table flows in the page-level scroll (MetricsLayout.Root scroll="page"); a sticky header - // keeps the column labels pinned as the whole column scrolls. + // No `stickyHeader` — it drops the table's own overflow-x-auto and the charts scroll with it. return tab === "sessions" ? ( <Suspense fallback={<TableLoading />}> <TypedAwait resolve={sessionList} errorElement={<TableLoading />}> @@ -367,7 +370,7 @@ function AgentContentArea({ sessions={list.sessions} filters={list.filters} hasFilters={list.hasFilters} - stickyHeader + showTopBorder={false} /> ) : ( <TableLoading /> @@ -386,7 +389,7 @@ function AgentContentArea({ filters={list.filters} runs={list.runs} variant="dimmed" - stickyHeader + showTopBorder={false} /> ) : ( <TableLoading /> diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents/route.tsx index f6723ddebaa..9c8c2c8fa5c 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents/route.tsx @@ -1,10 +1,6 @@ import { Outlet } from "@remix-run/react"; -import { PageContainer } from "~/components/layout/AppLayout"; +// No PageContainer — the child pages render their own; nesting two collapses the inner one's height. export default function Page() { - return ( - <PageContainer> - <Outlet /> - </PageContainer> - ); + return <Outlet />; } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx index 6d81de239fe..b3a2b852ed9 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.scheduled.$taskParam/route.tsx @@ -28,6 +28,7 @@ import { DialogTrigger, } from "~/components/primitives/Dialog"; import { Header2 } from "~/components/primitives/Headers"; +import { TitleBar } from "~/components/primitives/TitleBar"; import { InfoPanel } from "~/components/primitives/InfoPanel"; import { NavBar, PageTitle } from "~/components/primitives/PageHeader"; import { PaginationControls } from "~/components/primitives/Pagination"; @@ -292,31 +293,15 @@ export default function Page() { <ResizablePanelGroup orientation="horizontal" className="max-h-full"> <ResizablePanel id="scheduled-task-main" min="300px"> <div className="grid h-full grid-rows-[auto_1fr_auto] overflow-hidden"> - {/* Top bar — title on the left; actions + TimeFilter + pagination on the right. - h-10 matches the right-hand sidebar header height. */} - <div className="flex min-h-10 items-center gap-2 border-b border-grid-dimmed bg-background-bright py-2 pl-3 pr-2"> - <Header2>Runs</Header2> + <div className="flex min-h-10 items-center gap-2 border-b border-grid-dimmed bg-background-bright px-2 py-2"> + <TimeFilter defaultPeriod="7d" labelName="Runs" /> <div className="ml-auto flex flex-wrap items-center justify-end gap-1.5"> - <CreateScheduleButton - isAtLimit={isAtLimit} - limits={limits} - canUpgrade={canUpgrade} - canPurchaseSchedules={scheduleList?.canPurchaseSchedules ?? false} - extraSchedules={scheduleList?.extraSchedules ?? 0} - maxScheduleQuota={scheduleList?.maxScheduleQuota ?? 0} - planScheduleLimit={scheduleList?.planScheduleLimit ?? 0} - schedulePricing={scheduleList?.schedulePricing ?? null} - onCreate={openCreateSchedule} - disabled={isCreatingSchedule} - /> - {newRunsCount > 0 ? ( - <NewRunsButton count={newRunsCount} onClick={() => showNewRunsRef.current()} /> - ) : null} - <TimeFilter defaultPeriod="7d" labelName="Runs" /> <LinkButton variant="secondary/small" to={v3RunsPath(organization, project, environment, filters)} LeadingIcon={RunsIcon} + // -7px halves the icon's default 6px of space on each side. + leadingIconClassName="-mx-[7px]" > View all runs </LinkButton> @@ -335,11 +320,18 @@ export default function Page() { > Bulk replay… </LinkButton> - <Suspense fallback={null}> - <TypedAwait resolve={runList} errorElement={null}> - {(list) => (list ? <ListPagination list={list} /> : null)} - </TypedAwait> - </Suspense> + <CreateScheduleButton + isAtLimit={isAtLimit} + limits={limits} + canUpgrade={canUpgrade} + canPurchaseSchedules={scheduleList?.canPurchaseSchedules ?? false} + extraSchedules={scheduleList?.extraSchedules ?? 0} + maxScheduleQuota={scheduleList?.maxScheduleQuota ?? 0} + planScheduleLimit={scheduleList?.planScheduleLimit ?? 0} + schedulePricing={scheduleList?.schedulePricing ?? null} + onCreate={openCreateSchedule} + disabled={isCreatingSchedule} + /> </div> </div> @@ -363,23 +355,39 @@ export default function Page() { {/* Runs table */} <ResizablePanel id="scheduled-task-content" min="160px"> - <div className="h-full overflow-hidden"> - <Suspense fallback={<TableLoading />}> - <TypedAwait resolve={runList} errorElement={<TableLoading />}> - {(list) => - list ? ( - <TaskRunsList - list={list} - taskSlug={task.slug} - onNewRunsCountChange={setNewRunsCount} - showNewRunsRef={showNewRunsRef} - /> - ) : ( - <TableLoading /> - ) - } - </TypedAwait> - </Suspense> + <div className="grid h-full grid-rows-[auto_1fr] overflow-hidden"> + {/* -mt-px absorbs the spare pixel below the handle's rule, centring the title. */} + <TitleBar title="Runs" className="-mt-px"> + {newRunsCount > 0 ? ( + <NewRunsButton + count={newRunsCount} + onClick={() => showNewRunsRef.current()} + /> + ) : null} + <Suspense fallback={null}> + <TypedAwait resolve={runList} errorElement={null}> + {(list) => (list ? <ListPagination list={list} /> : null)} + </TypedAwait> + </Suspense> + </TitleBar> + <div className="min-h-0 overflow-hidden"> + <Suspense fallback={<TableLoading />}> + <TypedAwait resolve={runList} errorElement={<TableLoading />}> + {(list) => + list ? ( + <TaskRunsList + list={list} + taskSlug={task.slug} + onNewRunsCountChange={setNewRunsCount} + showNewRunsRef={showNewRunsRef} + /> + ) : ( + <TableLoading /> + ) + } + </TypedAwait> + </Suspense> + </div> </div> </ResizablePanel> </ResizablePanelGroup> diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx index 12227dd66fd..15df31370d2 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.tasks.standard.$taskParam/route.tsx @@ -20,6 +20,7 @@ import { statusColor } from "~/components/primitives/charts/statusColors"; import { CopyableText } from "~/components/primitives/CopyableText"; import { DateTime } from "~/components/primitives/DateTime"; import { Header2 } from "~/components/primitives/Headers"; +import { TitleBar } from "~/components/primitives/TitleBar"; import { NavBar, PageTitle } from "~/components/primitives/PageHeader"; import { Paragraph } from "~/components/primitives/Paragraph"; import * as Property from "~/components/primitives/PropertyTable"; @@ -234,21 +235,8 @@ export default function Page() { <ResizablePanelGroup orientation="horizontal" className="max-h-full"> <ResizablePanel id="task-main" min="300px"> <div className="grid h-full grid-rows-[auto_1fr] overflow-hidden"> - {/* Top bar — title on the left; TimeFilter + pagination on the right. - h-10 matches the right-hand sidebar header height. */} - <div className="flex h-10 items-center border-b border-grid-dimmed bg-background-bright pl-3 pr-2"> - <Header2>Runs</Header2> - <div className="ml-auto flex items-center gap-1.5"> - {newRunsCount > 0 ? ( - <NewRunsButton count={newRunsCount} onClick={() => showNewRunsRef.current()} /> - ) : null} - <TimeFilter defaultPeriod="7d" labelName="Runs" /> - <Suspense fallback={null}> - <TypedAwait resolve={runList} errorElement={null}> - {(list) => (list ? <ListPagination list={list} /> : null)} - </TypedAwait> - </Suspense> - </div> + <div className="flex h-10 items-center border-b border-grid-dimmed bg-background-bright px-2"> + <TimeFilter defaultPeriod="7d" labelName="Runs" /> </div> <ResizablePanelGroup orientation="vertical" className="max-h-full"> @@ -263,23 +251,39 @@ export default function Page() { {/* Runs table */} <ResizablePanel id="task-content" min="160px"> - <div className="h-full overflow-hidden"> - <Suspense fallback={<TableLoading />}> - <TypedAwait resolve={runList} errorElement={<TableLoading />}> - {(list) => - list ? ( - <TaskRunsList - list={list} - taskSlug={task.slug} - onNewRunsCountChange={setNewRunsCount} - showNewRunsRef={showNewRunsRef} - /> - ) : ( - <TableLoading /> - ) - } - </TypedAwait> - </Suspense> + <div className="grid h-full grid-rows-[auto_1fr] overflow-hidden"> + {/* -mt-px absorbs the spare pixel below the handle's rule, centring the title. */} + <TitleBar title="Runs" className="-mt-px"> + {newRunsCount > 0 ? ( + <NewRunsButton + count={newRunsCount} + onClick={() => showNewRunsRef.current()} + /> + ) : null} + <Suspense fallback={null}> + <TypedAwait resolve={runList} errorElement={null}> + {(list) => (list ? <ListPagination list={list} /> : null)} + </TypedAwait> + </Suspense> + </TitleBar> + <div className="min-h-0 overflow-hidden"> + <Suspense fallback={<TableLoading />}> + <TypedAwait resolve={runList} errorElement={<TableLoading />}> + {(list) => + list ? ( + <TaskRunsList + list={list} + taskSlug={task.slug} + onNewRunsCountChange={setNewRunsCount} + showNewRunsRef={showNewRunsRef} + /> + ) : ( + <TableLoading /> + ) + } + </TypedAwait> + </Suspense> + </div> </div> </ResizablePanel> </ResizablePanelGroup> From 8f66af6e18b73ceaf4a8c2d198bb662a7bb85202 Mon Sep 17 00:00:00 2001 From: Iss <74388823+isshaddad@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:49:09 -0400 Subject: [PATCH 57/67] fix(webapp): stop the sidebar feedback popover from canceling the submit (#4445) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Help & Feedback → "Contact us" form in the sidebar intermittently failed to send. The `<Feedback>` dialog was nested inside the Help popover, so clicking **Send** closed the popover and unmounted the form mid-submit — canceling the `POST /resources/feedback` before it went out. The message was silently lost (the success toast still shows). A race, so it "worked sometimes"; the standalone "I'm stuck!" path was unaffected. **Fix:** host the Feedback dialog *outside* the popover (same pattern as `AskAIRoot`) and open it from the menu item, so closing the popover no longer tears down the form. `Feedback` gains an optional controlled `open`/`setOpen` mode; existing `button`-triggered usages are unchanged. ## Changes - `Feedback.tsx` — optional controlled `open`/`setOpen`; `button` now optional. - `HelpAndFeedbackPopover.tsx` — "Contact us…" opens a `<Feedback>` hosted outside `PopoverContent`. - `.server-changes/fix-sidebar-feedback.md` — user-facing note. ## Testing Webapp typecheck passes. Sidebar "Contact us…" now sends on every attempt (Network: `POST /resources/feedback` → `204`, never `(canceled)`); "I'm stuck!" and the `?feedbackPanel=` open path unchanged. --- .server-changes/fix-sidebar-feedback.md | 6 +++ apps/webapp/app/components/Feedback.tsx | 37 ++++++++++++++++--- .../navigation/HelpAndFeedbackPopover.tsx | 22 +++++++---- 3 files changed, 51 insertions(+), 14 deletions(-) create mode 100644 .server-changes/fix-sidebar-feedback.md diff --git a/.server-changes/fix-sidebar-feedback.md b/.server-changes/fix-sidebar-feedback.md new file mode 100644 index 00000000000..066332c0314 --- /dev/null +++ b/.server-changes/fix-sidebar-feedback.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Fix the sidebar "Help & Feedback → Contact us" form sometimes not sending your message. It now sends reliably every time. diff --git a/apps/webapp/app/components/Feedback.tsx b/apps/webapp/app/components/Feedback.tsx index ebd61180de5..b679b042da1 100644 --- a/apps/webapp/app/components/Feedback.tsx +++ b/apps/webapp/app/components/Feedback.tsx @@ -9,7 +9,7 @@ import { parseWithZod } from "@conform-to/zod"; import { InformationCircleIcon, ArrowUpCircleIcon } from "@heroicons/react/20/solid"; import { EnvelopeIcon, ShieldCheckIcon } from "@heroicons/react/24/solid"; import { Form, useActionData, useLocation, useNavigation, useSearchParams } from "@remix-run/react"; -import { type ReactNode, useEffect, useState } from "react"; +import { type ReactNode, useEffect, useRef, useState } from "react"; import { type FeedbackType, feedbackTypes, schema } from "~/routes/resources.feedback"; import { Button } from "./primitives/Buttons"; import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "./primitives/Dialog"; @@ -27,13 +27,26 @@ import { TextLink } from "./primitives/TextLink"; import { DialogClose } from "@radix-ui/react-dialog"; type FeedbackProps = { - button: ReactNode; + button?: ReactNode; defaultValue?: FeedbackType; onOpenChange?: (open: boolean) => void; -}; +} & + // Controlled mode is all-or-none: pass both open + setOpen to host the dialog outside a popover + // (so the popover closing can't unmount the form mid-submit and cancel the feedback POST), or + // neither for the self-managed, button-triggered dialog. Passing only one is a broken half-state. + ({ open?: never; setOpen?: never } | { open: boolean; setOpen: (open: boolean) => void }); -export function Feedback({ button, defaultValue = "bug", onOpenChange }: FeedbackProps) { - const [open, setOpen] = useState(false); +export function Feedback({ + button, + defaultValue = "bug", + onOpenChange, + open: openProp, + setOpen: setOpenProp, +}: FeedbackProps) { + const [openState, setOpenState] = useState(false); + // Controlled when the caller passes open/setOpen (hosted outside a popover); otherwise self-managed. + const open = openProp ?? openState; + const setOpen = setOpenProp ?? setOpenState; const [searchParams, setSearchParams] = useSearchParams(); const location = useLocation(); const lastSubmission = useActionData(); @@ -72,6 +85,18 @@ export function Feedback({ button, defaultValue = "bug", onOpenChange }: Feedbac } }, [searchParams]); + // Reset the topic to the default once the dialog closes, so reopening always starts fresh. The + // dialog is now persistently mounted (hosted outside the popover), so without this it would keep + // the previously chosen topic selected and risk filing feedback under the wrong category. Keyed + // on the close transition (not just `!open`) so the ?feedbackPanel= open path isn't clobbered. + const wasOpen = useRef(open); + useEffect(() => { + if (wasOpen.current && !open) { + setType(defaultValue); + } + wasOpen.current = open; + }, [open, defaultValue]); + const handleOpenChange = (value: boolean) => { setOpen(value); onOpenChange?.(value); @@ -79,7 +104,7 @@ export function Feedback({ button, defaultValue = "bug", onOpenChange }: Feedbac return ( <Dialog open={open} onOpenChange={handleOpenChange}> - <DialogTrigger asChild>{button}</DialogTrigger> + {button ? <DialogTrigger asChild>{button}</DialogTrigger> : null} <DialogContent> <DialogHeader>Contact us</DialogHeader> <div className="mt-2 flex flex-col gap-4"> diff --git a/apps/webapp/app/components/navigation/HelpAndFeedbackPopover.tsx b/apps/webapp/app/components/navigation/HelpAndFeedbackPopover.tsx index 669271fe766..c7276dd5bc6 100644 --- a/apps/webapp/app/components/navigation/HelpAndFeedbackPopover.tsx +++ b/apps/webapp/app/components/navigation/HelpAndFeedbackPopover.tsx @@ -35,6 +35,10 @@ export function HelpAndFeedback({ projectId?: string; }) { const [isHelpMenuOpen, setHelpMenuOpen] = useState(false); + // Hosted outside the popover (below) and opened from the menu item, so the popover closing never + // unmounts the feedback form mid-submit — that teardown was intermittently canceling the POST to + // /resources/feedback, so messages sent from the sidebar were silently lost. + const [isFeedbackOpen, setFeedbackOpen] = useState(false); const _currentPlan = useCurrentPlan(); const { changelogs } = useRecentChangelogs(organizationId, projectId); @@ -164,14 +168,14 @@ export function HelpAndFeedback({ target="_blank" /> <Shortcuts /> - <Feedback - button={ - <SideMenuItemButton - icon={EnvelopeIcon} - name="Contact us…" - data-action="contact-us" - /> - } + <SideMenuItemButton + icon={EnvelopeIcon} + name="Contact us…" + data-action="contact-us" + onClick={() => { + setHelpMenuOpen(false); + setFeedbackOpen(true); + }} /> </div> <div className="flex flex-col gap-1 p-1"> @@ -206,6 +210,8 @@ export function HelpAndFeedback({ </Popover> )} </AskAIRoot> + {/* Hosted outside the popover so closing the menu can't unmount the form mid-submit. */} + <Feedback open={isFeedbackOpen} setOpen={setFeedbackOpen} /> </motion.div> ); } From 5f29ae49ab87f99e642a3f62c965cef8550c9fa0 Mon Sep 17 00:00:00 2001 From: Eric Allam <eric@trigger.dev> Date: Mon, 3 Aug 2026 10:09:34 +0100 Subject: [PATCH 58/67] feat(webapp): default the queue metrics period to 1 hour and remember it (#4438) ## Summary The Queues list and queue detail pages opened on a 1 day window, and went back to it every time you navigated between queues or reloaded. They now default to the last hour, and the period you pick is remembered across navigations and refreshes. ## Design The last period is stored in a `queueMetricsPeriod` cookie, written client-side whenever a `period` lands in the URL and read by both loaders. A cookie rather than localStorage because the queues list renders its per-queue metrics columns server-side: with localStorage the page would paint the 1 hour default and then re-fetch, and the picker would flash the wrong window. Both pages resolve the window once, in one place, and pass it down: ```ts period: resolveQueueMetricsPeriod({ period: value("period"), // a usable period in the URL wins from: value("from"), // an absolute range means "no period" to: value("to"), defaultPeriod, // otherwise the remembered default from the loader }), ``` That keeps the picker pill and every chart query on the same value, so no call site falls back to its own default. Periods the picker could never produce (a hand-edited `?period=garbage`, or a window past the 30 day retention) fall back to the default, and the picker renders the resolved window rather than the raw search param so the label can't disagree with the data. Absolute from/to ranges, including drag-to-zoom, are not remembered, since they would pin later visits to a window that has gone stale. While wiring that up: the two queue-metric queries that go straight to ClickHouse (the list table and the concurrency-keys endpoint) never applied the org's `queryPeriodDays` limit, so a hand-typed `?period=` read further back than the plan allows. Everything behind `/resources/metric` is already clipped that way by `executeQuery`; both of these now clip with the same limit, capped at the retention window, and the plan cap is resolved once per load and handed to the page instead of each route deriving its own copy from the client-side subscription. Verified on both pages: default with no cookie is 1 hr, picking 6 hrs survives navigating away and back to a param-free URL and a hard reload, clearing the cookie returns to 1 hr, an oversized period falls back without being remembered, and an absolute range still renders as a range. --- .../queue-metrics-remember-period.md | 6 + .../components/queues/QueueMetricCards.tsx | 3 +- .../queues/queueMetricsPeriod.server.ts | 31 +++++ .../components/queues/queueMetricsPeriod.ts | 131 ++++++++++++++++++ .../route.tsx | 57 ++++++-- .../route.tsx | 32 +++-- .../resources.queues.concurrency-keys.ts | 23 +-- 7 files changed, 250 insertions(+), 33 deletions(-) create mode 100644 .server-changes/queue-metrics-remember-period.md create mode 100644 apps/webapp/app/components/queues/queueMetricsPeriod.server.ts create mode 100644 apps/webapp/app/components/queues/queueMetricsPeriod.ts diff --git a/.server-changes/queue-metrics-remember-period.md b/.server-changes/queue-metrics-remember-period.md new file mode 100644 index 00000000000..339d3139c8d --- /dev/null +++ b/.server-changes/queue-metrics-remember-period.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +The Queues pages now open on the last hour instead of the last day, and remember the time period you picked when you navigate between queues or reload the page. diff --git a/apps/webapp/app/components/queues/QueueMetricCards.tsx b/apps/webapp/app/components/queues/QueueMetricCards.tsx index 4349ea80c36..95dabff0d71 100644 --- a/apps/webapp/app/components/queues/QueueMetricCards.tsx +++ b/apps/webapp/app/components/queues/QueueMetricCards.tsx @@ -15,6 +15,7 @@ import { Header3 } from "~/components/primitives/Headers"; import { Paragraph } from "~/components/primitives/Paragraph"; import { InfoIconTooltip } from "~/components/primitives/Tooltip"; import { useSearchParams } from "~/hooks/useSearchParam"; +import { QUEUE_METRICS_DEFAULT_PERIOD } from "~/components/queues/queueMetricsPeriod"; import { cn } from "~/utils/cn"; import { formatNumberCompact } from "~/utils/numberFormatter"; @@ -34,8 +35,6 @@ export const QUEUE_METRIC_COLORS = { ckWait: "#F59E0B", }; -export const QUEUE_METRICS_DEFAULT_PERIOD = "1d"; - export type QueueMetricIds = { organizationId: string; projectId: string; diff --git a/apps/webapp/app/components/queues/queueMetricsPeriod.server.ts b/apps/webapp/app/components/queues/queueMetricsPeriod.server.ts new file mode 100644 index 00000000000..fa50e464a4f --- /dev/null +++ b/apps/webapp/app/components/queues/queueMetricsPeriod.server.ts @@ -0,0 +1,31 @@ +import { getCachedLimit } from "~/services/platform.v3.server"; +import { logger } from "~/services/logger.server"; +import { QUEUE_METRICS_RETENTION_DAYS } from "./queueMetricsPeriod"; + +/** + * The furthest back this org can query queue metrics: their plan's query period, capped at the + * 30 day retention. Same limit `executeQuery` enforces, so the queue-metric queries that bypass it + * and go straight to ClickHouse stay in step with the ones that don't. + * + * Read through the limit cache: the queues page revalidates on an interval, so this runs far more + * often than a one-off page load. Never throws, so a cache or platform outage costs the caller its + * time filter rather than the whole page: the retention cap is the widest window the data can cover + * anyway, and the queries stay tenant-scoped either way. + */ +export async function queueMetricsMaxPeriodDays(organizationId: string): Promise<number> { + try { + const cached = await getCachedLimit( + organizationId, + "queryPeriodDays", + QUEUE_METRICS_RETENTION_DAYS + ); + const planPeriodDays = cached.val ?? QUEUE_METRICS_RETENTION_DAYS; + return Math.min(planPeriodDays, QUEUE_METRICS_RETENTION_DAYS); + } catch (error) { + logger.warn("Queue metrics query period limit unavailable, falling back to retention", { + organizationId, + error, + }); + return QUEUE_METRICS_RETENTION_DAYS; + } +} diff --git a/apps/webapp/app/components/queues/queueMetricsPeriod.ts b/apps/webapp/app/components/queues/queueMetricsPeriod.ts new file mode 100644 index 00000000000..d18bdbc2fce --- /dev/null +++ b/apps/webapp/app/components/queues/queueMetricsPeriod.ts @@ -0,0 +1,131 @@ +import parse from "parse-duration"; +import { useEffect } from "react"; + +/** + * The time window the queue-metrics pages (queues list + queue detail) use when the URL carries + * no explicit period, and the memory that makes the user's last pick stick. + * + * The last period picked is stored in a cookie rather than localStorage so the loaders can read it + * and the first render already uses the remembered window (with localStorage the page would paint + * the default and then re-fetch). Absolute from/to ranges are never remembered: they'd pin later + * visits to a window that goes stale. + */ +export const QUEUE_METRICS_DEFAULT_PERIOD = "1h"; + +const COOKIE_NAME = "queueMetricsPeriod"; +const COOKIE_MAX_AGE_SECONDS = 60 * 60 * 24 * 365; + +/** + * The shape TimeFilter writes: a count plus a minute/hour/day unit. The count is unbounded here + * because the picker accepts any positive integer for a custom duration (`10000m` is a little under + * 7 days); the retention bound below is what rules a window out. + */ +const PERIOD_PATTERN = /^\d+[mhd]$/; + +/** Queue metrics are retained for 30 days, so a longer window can only ever render empty. */ +export const QUEUE_METRICS_RETENTION_DAYS = 30; + +const MINUTE_MS = 60 * 1000; +const HOUR_MS = 60 * MINUTE_MS; +const DAY_MS = 24 * HOUR_MS; +const MAX_PERIOD_MS = QUEUE_METRICS_RETENTION_DAYS * DAY_MS; + +function isPeriod(value: string | undefined | null): value is string { + if (typeof value !== "string" || !PERIOD_PATTERN.test(value)) return false; + const ms = parse(value); + return typeof ms === "number" && ms > 0 && ms <= MAX_PERIOD_MS; +} + +/** Loader side: the remembered period, falling back to the default when nothing usable is stored. */ +export function queueMetricsPeriodFromRequest(request: Request): string { + const header = request.headers.get("cookie"); + if (!header) return QUEUE_METRICS_DEFAULT_PERIOD; + + for (const part of header.split(";")) { + const separator = part.indexOf("="); + if (separator === -1) continue; + if (part.slice(0, separator).trim() !== COOKIE_NAME) continue; + const value = part.slice(separator + 1).trim(); + return isPeriod(value) ? value : QUEUE_METRICS_DEFAULT_PERIOD; + } + + return QUEUE_METRICS_DEFAULT_PERIOD; +} + +/** + * Remember the period currently in the URL so the next visit to a queue-metrics page opens on it. + * Pass the raw `period` search param: an absent one (the page is on its default) or an absolute + * from/to range leaves the stored value alone. + */ +export function useRememberQueueMetricsPeriod(period: string | undefined) { + useEffect(() => { + if (!isPeriod(period)) return; + document.cookie = `${COOKIE_NAME}=${period}; path=/; max-age=${COOKIE_MAX_AGE_SECONDS}; samesite=lax`; + }, [period]); +} + +/** + * The window the page should show: a usable period in the URL wins, an absolute range means "no + * period", and everything else (including a period the picker could never produce, e.g. a + * hand-edited `?period=garbage`) falls back to the remembered default the loader resolved. The + * result is held inside the org's plan query period, since that is the window the data will cover. + * + * Both the loaders and the client-side chart queries resolve through here, so they can't disagree + * about the window. + */ +export function resolveQueueMetricsPeriod({ + period, + from, + to, + defaultPeriod, + maxPeriodDays, +}: { + period: string | undefined; + from: string | undefined; + to: string | undefined; + defaultPeriod: string; + maxPeriodDays: number; +}): string | null { + if (isPeriod(period)) return clampQueueMetricsPeriod(period, maxPeriodDays); + if (from || to) return null; + return clampQueueMetricsPeriod(defaultPeriod, maxPeriodDays); +} + +/** + * Hold a period inside a day budget (the org's plan query period). A period longer than the plan + * allows becomes the plan's maximum, so the picker shows the window the data covers. + * + * The budget is whatever the plan says, not necessarily a whole number of days, so the replacement + * is expressed in the largest unit that divides it: rounding down keeps the period inside the + * budget rather than a hair over it. + */ +export function clampQueueMetricsPeriod(period: string, maxPeriodDays: number): string { + const maxMs = maxPeriodDays * DAY_MS; + const ms = parse(period); + if (typeof ms === "number" && ms > 0 && ms <= maxMs) return period; + + const days = Math.floor(maxMs / DAY_MS); + if (days >= 1) return `${days}d`; + const hours = Math.floor(maxMs / HOUR_MS); + if (hours >= 1) return `${hours}h`; + return `${Math.max(1, Math.floor(maxMs / MINUTE_MS))}m`; +} + +/** + * Pull a window forward to the earliest time the org's plan can query, the same clip `executeQuery` + * applies to every metric query. Queue-metric queries that go straight to ClickHouse (the queues + * list table, the concurrency-keys endpoint) have to apply it themselves, otherwise a hand-typed + * `?period=` reaches further back than the plan allows. + * + * A range that ends before the plan's earliest queryable time collapses to an empty window rather + * than an inverted one, which is what the enforced lower bound in `executeQuery` yields for the + * same request: no rows. + */ +export function clipQueueMetricsWindow( + window: { from: Date; to: Date }, + maxPeriodDays: number +): { from: Date; to: Date } { + const earliest = new Date(Date.now() - maxPeriodDays * DAY_MS); + const from = window.from < earliest ? earliest : window.from; + return { from, to: window.to < from ? from : window.to }; +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 45c395b54f3..47ab9581695 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -102,6 +102,16 @@ import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; import { BigNumber } from "~/components/metrics/BigNumber"; import { canAccessQueueMetricsUi } from "~/v3/canAccessQueueMetricsUi.server"; import { QueueAllocationPresenter } from "~/presenters/v3/QueueAllocationPresenter.server"; +import { + QUEUE_METRICS_DEFAULT_PERIOD, + QUEUE_METRICS_RETENTION_DAYS, + clampQueueMetricsPeriod, + clipQueueMetricsWindow, + queueMetricsPeriodFromRequest, + resolveQueueMetricsPeriod, + useRememberQueueMetricsPeriod, +} from "~/components/queues/queueMetricsPeriod"; +import { queueMetricsMaxPeriodDays } from "~/components/queues/queueMetricsPeriod.server"; const SearchParamsSchema = z.object({ query: z.string().optional(), @@ -112,8 +122,6 @@ const SearchParamsSchema = z.object({ sort: z.enum(["busiest", "queued", "name"]).optional(), }); -const QUEUE_METRICS_DEFAULT_PERIOD = "1d"; - // The live "Queued" / "Running" header blocks poll ClickHouse on a short cadence so they stay // current after first paint. They read the env-wide gauges from env_metrics (the env-level rollup // of queue_metrics, cheapest for a dimension-free query), always over a fixed 15m window regardless @@ -163,6 +171,14 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { // no metrics query fires. const queueMetricsUiEnabled = await canAccessQueueMetricsUi({ userId, organizationSlug }); + const maxPeriodDays = queueMetricsUiEnabled + ? await queueMetricsMaxPeriodDays(environment.organizationId) + : QUEUE_METRICS_RETENTION_DAYS; + const defaultPeriod = clampQueueMetricsPeriod( + queueMetricsPeriodFromRequest(request), + maxPeriodDays + ); + try { const queueListPresenter = new QueueListPresenter(); const queues = await queueListPresenter.call({ @@ -194,12 +210,17 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const queueNames = queues.queues.map((q) => q.type === "task" ? `task/${q.name}` : q.name ); - const timeRange = timeFilterFromTo({ - period, - from: parseFiniteInt(from), - to: parseFiniteInt(to), - defaultPeriod: QUEUE_METRICS_DEFAULT_PERIOD, - }); + const timeRange = clipQueueMetricsWindow( + timeFilterFromTo({ + period: + resolveQueueMetricsPeriod({ period, from, to, defaultPeriod, maxPeriodDays }) ?? + undefined, + from: parseFiniteInt(from), + to: parseFiniteInt(to), + defaultPeriod, + }), + maxPeriodDays + ); const queueMetrics = queueNames.length > 0 ? await presenter.getQueueListMetrics({ @@ -239,6 +260,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { metrics, allocation, queueMetricsUiEnabled, + defaultPeriod, + maxPeriodDays, }); } catch (error) { console.error(error); @@ -362,6 +385,8 @@ function QueuesWithMetricsView() { autoReloadPollIntervalMs, metrics, allocation, + defaultPeriod, + maxPeriodDays, } = useTypedLoaderData<typeof loader>(); const metricsByQueue = metrics?.byQueue ?? {}; @@ -377,18 +402,21 @@ function QueuesWithMetricsView() { const project = useProject(); const env = useEnvironment(); const plan = useCurrentPlan(); - // Queue metrics are retained for 30 days in ClickHouse, so cap the picker there even for - // plans whose query-period limit was raised above it — a longer window would render empty. - const planPeriodDays = plan?.v3Subscription?.plan?.limits?.queryPeriodDays?.number; - const maxPeriodDays = Math.min(planPeriodDays ?? 30, 30); // The header tiles fetch client-side with the same period/from/to the TimeFilter writes. const { value } = useSearchParams(); const timeRange = { - period: value("period") ?? null, + period: resolveQueueMetricsPeriod({ + period: value("period"), + from: value("from"), + to: value("to"), + defaultPeriod, + maxPeriodDays, + }), from: value("from") ?? null, to: value("to") ?? null, }; + useRememberQueueMetricsPeriod(value("period")); useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true }); @@ -473,7 +501,8 @@ function QueuesWithMetricsView() { </div> <div className="flex items-center gap-1.5"> <TimeFilter - defaultPeriod={QUEUE_METRICS_DEFAULT_PERIOD} + period={timeRange.period ?? undefined} + defaultPeriod={defaultPeriod} labelName="Period" maxPeriodDays={maxPeriodDays} shortcut={{ key: "d" }} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 8dc0c2c24f4..37f955cd9d5 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -22,7 +22,6 @@ import { ChartSyncProvider } from "~/components/primitives/charts/ChartSyncConte import { useZoomToTimeFilter } from "~/hooks/useZoomToTimeFilter"; import { QUEUE_METRIC_COLORS as COLORS, - QUEUE_METRICS_DEFAULT_PERIOD, QueueMetricChartCard as QueueDetailChartCard, type QueueMetricIds as Ids, type QueueMetricTimeRange as TimeRangeParams, @@ -55,7 +54,6 @@ import type { ConcurrencyKeyRow, ConcurrencyKeysResponse, } from "~/routes/resources.queues.concurrency-keys"; -import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; import { canAccessQueueMetricsUi } from "~/v3/canAccessQueueMetricsUi.server"; import { requireUserId } from "~/services/session.server"; import { docsPath, EnvironmentParamSchema, v3RunsPath } from "~/utils/pathBuilder"; @@ -67,6 +65,13 @@ import { QueueOverrideConcurrencyButton, QueuePauseResumeButton, } from "~/components/queues/QueueControls"; +import { + clampQueueMetricsPeriod, + queueMetricsPeriodFromRequest, + resolveQueueMetricsPeriod, + useRememberQueueMetricsPeriod, +} from "~/components/queues/queueMetricsPeriod"; +import { queueMetricsMaxPeriodDays } from "~/components/queues/queueMetricsPeriod.server"; import { LinkButton } from "~/components/primitives/Buttons"; import { RunsIcon } from "~/assets/icons/RunsIcon"; import { InfoPanel } from "~/components/primitives/InfoPanel"; @@ -106,6 +111,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const queue = retrieve.queue; const fullName = queue.type === "task" ? `task/${queue.name}` : queue.name; + const maxPeriodDays = await queueMetricsMaxPeriodDays(environment.organizationId); + const [ckBreakdown, oldestQueuedAt] = await Promise.all([ engine.concurrencyKeyBreakdown(environment, fullName, { limit: CK_LIVE_LIMIT }), // Enqueue time of the oldest run still waiting in the queue right now (any queue, keyed or @@ -134,6 +141,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { oldestQueuedAt: oldestQueuedAt ?? null, loadedAt: Date.now(), backPath: url.pathname.replace(/\/[^/]+$/, ""), + defaultPeriod: clampQueueMetricsPeriod(queueMetricsPeriodFromRequest(request), maxPeriodDays), + maxPeriodDays, ids: { organizationId: environment.organizationId, projectId: environment.projectId, @@ -210,19 +219,23 @@ export default function Page() { loadedAt, backPath, ids, + defaultPeriod, + maxPeriodDays, } = useTypedLoaderData<typeof loader>(); - const plan = useCurrentPlan(); - // Queue metrics are retained for 30 days in ClickHouse, so cap the picker there even for - // plans whose query-period limit was raised above it — a longer window would render empty. - const planPeriodDays = plan?.v3Subscription?.plan?.limits?.queryPeriodDays?.number; - const maxPeriodDays = Math.min(planPeriodDays ?? 30, 30); const { value, replace } = useSearchParams(); const timeRange: TimeRangeParams = { - period: value("period") ?? null, + period: resolveQueueMetricsPeriod({ + period: value("period"), + from: value("from"), + to: value("to"), + defaultPeriod, + maxPeriodDays, + }), from: value("from") ?? null, to: value("to") ?? null, }; + useRememberQueueMetricsPeriod(value("period")); // The Concurrency keys tab exists only for queues with key activity: live keys in the // ckIndex, or nonzero CK history in the selected range (one cached scalar query decides). @@ -283,7 +296,8 @@ export default function Page() { /> ) : null} <TimeFilter - defaultPeriod={QUEUE_METRICS_DEFAULT_PERIOD} + period={timeRange.period ?? undefined} + defaultPeriod={defaultPeriod} labelName="Period" maxPeriodDays={maxPeriodDays} shortcut={{ key: "d" }} diff --git a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts index 7d657a781c5..992ace788a5 100644 --- a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts +++ b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts @@ -1,6 +1,11 @@ import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; import { z } from "zod"; import { timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; +import { + QUEUE_METRICS_DEFAULT_PERIOD, + clipQueueMetricsWindow, +} from "~/components/queues/queueMetricsPeriod"; +import { queueMetricsMaxPeriodDays } from "~/components/queues/queueMetricsPeriod.server"; import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { findEnvironmentById, hasAccessToEnvironment } from "~/models/runtimeEnvironment.server"; import { requireUserId } from "~/services/session.server"; @@ -13,8 +18,7 @@ import { engine } from "~/v3/runEngine.server"; // Redis (O(page), independent of total key cardinality). This replaces the old top-50 cap. export const CONCURRENCY_KEYS_PER_PAGE = 25; -// Matches QUEUE_METRICS_DEFAULT_PERIOD (the detail page's TimeFilter default). -const DEFAULT_PERIOD = "1d"; +const DEFAULT_PERIOD = QUEUE_METRICS_DEFAULT_PERIOD; const Body = z.object({ organizationId: z.string(), @@ -110,12 +114,15 @@ export const action = async ({ request }: ActionFunctionArgs) => { return json<ConcurrencyKeysResponse>({ success: false, error: "Not found" }, { status: 404 }); } - const range = timeFilterFromTo({ - period: period ?? undefined, - from: from ?? undefined, - to: to ?? undefined, - defaultPeriod: DEFAULT_PERIOD, - }); + const range = clipQueueMetricsWindow( + timeFilterFromTo({ + period: period ?? undefined, + from: from ?? undefined, + to: to ?? undefined, + defaultPeriod: DEFAULT_PERIOD, + }), + await queueMetricsMaxPeriodDays(organizationId) + ); const startTime = formatClickhouseDateTime(new Date(floorToMinute(range.from.getTime()))); const endTime = formatClickhouseDateTime(new Date(ceilToMinute(range.to.getTime()))); From d9f4fea939451b763f094eb7d50c929c6db2a5e5 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:57:24 +0100 Subject: [PATCH 59/67] docs: restructure self-hosting kubernetes guide (#4481) Restructures the Kubernetes self-hosting guide around two explicit paths - an **evaluation install** (bundled datastores, one command) and a **production install** (external datastores, your own secrets) - so every configuration decision belongs to one path or the other instead of being a flat list of options with caveats. Also in this pass: - Adds an architecture overview (component-to-`values.yaml` map) and a post-install "verify it" step. - Consolidates the previously scattered upgrade notes into a single collapsible group, and cuts implementation detail and historical asides that no longer apply. - Removes a duplicated object-storage section (two configs in two styles) and trims the Docker ClickHouse note down to what a self-hoster needs to act on. --- docs/self-hosting/docker.mdx | 10 +- docs/self-hosting/env/webapp.mdx | 2 +- docs/self-hosting/kubernetes.mdx | 545 +++++++++++++++---------------- 3 files changed, 263 insertions(+), 294 deletions(-) diff --git a/docs/self-hosting/docker.mdx b/docs/self-hosting/docker.mdx index 065b4ad55da..a4fabfcc08d 100644 --- a/docs/self-hosting/docker.mdx +++ b/docs/self-hosting/docker.mdx @@ -365,12 +365,10 @@ We patch the latest released version line only, so keep an eye on new releases t You can also lock the versions of the bundled services, for example with `CLICKHOUSE_IMAGE_TAG`. If you do, or if you bring your own ClickHouse via `CLICKHOUSE_URL`, note that Trigger.dev requires ClickHouse 25.8 or newer. <Note> - The bundled ClickHouse now uses the official `clickhouse/clickhouse-server` image. Your existing - data volume carries over automatically. If you previously pinned `CLICKHOUSE_IMAGE_TAG` to a - Bitnami tag (for example `25.7.5-debian-12-r0`), update it to an official image tag such as - `26.2` — Bitnami tags don't exist in the official repository. Note the switch is one-way: the - official image takes ownership of the data files, so rolling back to the Bitnami image requires - manually restoring their previous owner (`chown -R 1001:1001` on the volume). + The bundled ClickHouse uses the official `clickhouse/clickhouse-server` image, and your existing + data volume carries over automatically on upgrade. If you pinned `CLICKHOUSE_IMAGE_TAG` to a + Bitnami tag, switch it to an official tag (for example `26.2`) — Bitnami tags don't exist in the + official repository. </Note> <Note> diff --git a/docs/self-hosting/env/webapp.mdx b/docs/self-hosting/env/webapp.mdx index 09416f0f38f..82dd6958a60 100644 --- a/docs/self-hosting/env/webapp.mdx +++ b/docs/self-hosting/env/webapp.mdx @@ -12,7 +12,7 @@ mode: "wide" | `MAGIC_LINK_SECRET` | Yes | — | Magic link encryption secret. Run: `openssl rand -hex 16` | | `ENCRYPTION_KEY` | Yes | — | Secret store encryption key. Run: `openssl rand -hex 16` | | `MANAGED_WORKER_SECRET` | Yes | — | Managed worker secret. Must be set and match supervisor. Run: `openssl rand -hex 32` | -| `ALLOW_INSECURE_DEFAULT_SECRETS` | No | false | Boot even if a secret is still a known-insecure published default. Temporary escape hatch for values you can't safely rotate yet (see [Secret generation and rotation](/self-hosting/kubernetes#secret-generation-and-rotation)). | +| `ALLOW_INSECURE_DEFAULT_SECRETS` | No | false | Boot even if a secret is still a known-insecure published default. Temporary escape hatch for values you can't safely rotate yet (see [Secrets](/self-hosting/kubernetes#secrets)). | | **Domains & ports** | | | | | `REMIX_APP_PORT` | No | 3030 | Remix app port. | | `APP_ORIGIN` | Yes | http://localhost:3030 | App origin URL. | diff --git a/docs/self-hosting/kubernetes.mdx b/docs/self-hosting/kubernetes.mdx index 1939ff5dfb8..2299373fd16 100644 --- a/docs/self-hosting/kubernetes.mdx +++ b/docs/self-hosting/kubernetes.mdx @@ -1,15 +1,41 @@ --- title: "Kubernetes" -description: "You can self-host Trigger.dev in Kubernetes using our official Helm chart." +description: "Deploy Trigger.dev to Kubernetes with our official Helm chart - one command for evaluation, external datastores for production." --- -The following instructions will help you deploy Trigger.dev to Kubernetes using our official Helm chart. Make sure to read the self-hosting [overview](/self-hosting/overview) first. +The official Helm chart installs the full Trigger.dev stack into a Kubernetes namespace. Read the self-hosting [overview](/self-hosting/overview) first. -As self-hosted deployments tend to have unique requirements and configurations, we don't provide specific advice for securing your deployment, scaling up, or improving reliability. +Pick a path and stay on it. Run the **evaluation install** to try Trigger.dev with everything bundled in-cluster - one `helm install`, default values, no external dependencies. Run the **production install** for real workloads: external Postgres, Redis, ClickHouse, and object storage, your own secrets, and HA on the datastores you already operate. Each configuration decision below belongs to one path or the other. -Should the burden ever get too much, we'd be happy to see you on [Trigger.dev cloud](https://trigger.dev/pricing) where we deal with these concerns for you. +This guide alone is unlikely to produce a hardened production deployment - securing, scaling, and reliability-tuning your cluster remain yours to own. Should the burden get too much, we'd be happy to see you on [Trigger.dev cloud](https://trigger.dev/pricing) where we deal with these concerns for you. -**Warning:** This guide alone is unlikely to result in a production-ready deployment. Security, scaling, and reliability concerns are not fully addressed here. +## Architecture + +The chart deploys a webapp, a supervisor, and the datastores they depend on. The supervisor pulls runs from the webapp and schedules each one onto a worker node, which executes the task in a container sized by its machine preset. + +```mermaid +flowchart TD + webapp["webapp<br/>API + dashboard"] + supervisor["supervisor"] + workers["worker nodes<br/>task containers"] + pg[("Postgres<br/>primary DB")] + redis[("Redis<br/>queue / state")] + ch[("ClickHouse<br/>task events v2")] + obj[("object storage<br/>MinIO / external S3")] + reg[("registry<br/>deploy images")] + s2[("s2 / s2-lite<br/>realtime streams v2")] + + webapp --> pg + webapp --> redis + webapp --> ch + webapp --> obj + webapp --> s2 + webapp -->|worker token| supervisor + supervisor --> workers + workers --> reg +``` + +Each component maps to a top-level key in `values.yaml`: `webapp`, `supervisor`, `postgres`, `redis`, `electric`, `clickhouse`, `s3` (object storage), `s2`, and `registry`. The datastores default to `deploy: true` and run in-cluster; set `deploy: false` with an `external` block to bring your own. The registry is the exception - it defaults to `deploy: false`, so you point it at an external one. Webapp settings live under `webapp`, supervisor settings under `supervisor.config`, and each service takes `extraEnvVars` for anything else. ## Requirements @@ -52,107 +78,111 @@ webapp: memory: 4Gi ``` -## Installation - -### Quick start +## Evaluation install -1. Install with default values (for testing only): +Bundled datastores let you run the whole stack with a single command and no external services. The default values are insecure and are only suitable for testing - for real workloads, follow the [production install](#production-install) instead. -```bash -helm upgrade -n trigger --install trigger \ - oci://ghcr.io/triggerdotdev/charts/trigger \ - --version "^4.5.0" \ - --create-namespace -``` - -2. Access the webapp: - -```bash -kubectl port-forward svc/trigger-webapp 3040:3030 -n trigger -``` +<Steps> + <Step title="Install the chart with default values"> + ```bash + helm upgrade -n trigger --install trigger \ + oci://ghcr.io/triggerdotdev/charts/trigger \ + --version "^4.5.0" \ + --create-namespace + ``` -3. Open the dashboard: `http://localhost:3040` + The chart deploys the webapp, supervisor, and bundled Postgres, Redis, Electric, ClickHouse, MinIO, and s2-lite. The registry is not deployed by default - configure an external one before deploying tasks. + </Step> -4. Login with the magic link: + <Step title="Port-forward the webapp"> + ```bash + kubectl port-forward svc/trigger-webapp 3040:3030 -n trigger + ``` -```bash -# Check the webapp logs -kubectl logs -n trigger deployment/trigger-webapp | grep -A1 "magic link" -``` + The dashboard is now reachable at `http://localhost:3040`. + </Step> -## Upgrading + <Step title="Log in with the magic link"> + Default installs print the login link to the webapp logs instead of sending email. -`helm upgrade` keeps generated secrets, PVCs, and datastore volumes, so most upgrades need no preparation. These chart versions are the exceptions. + ```bash + kubectl logs -n trigger deployment/trigger-webapp | grep -A1 "magic link" + ``` -### 4.5.6: two new required secret keys + Open the printed link to sign in. + </Step> +</Steps> -The webapp now reads `PROVIDER_SECRET` and `COORDINATOR_SECRET`. When you supply `secrets.existingSecret` the chart generates nothing and reads every key from your Secret, so both keys must exist there before you upgrade. Inline `secrets.*` values are ignored while `existingSecret` is set. +### Verify it -Add them with two fresh 32-character hex values: +Confirm every component reached a healthy state: ```bash -kubectl patch secret my-trigger-secrets -n trigger --type merge -p \ - "{\"stringData\":{\"PROVIDER_SECRET\":\"$(openssl rand -hex 16)\",\"COORDINATOR_SECRET\":\"$(openssl rand -hex 16)\"}}" +kubectl get pods -n trigger ``` -The chart checks your Secret in a pre-upgrade hook and aborts with the list of missing keys, leaving the running release untouched. Without that check a missing key surfaces as a `CreateContainerConfigError` partway through the webapp rollout. +The webapp, supervisor, and each bundled datastore pod should report `Running` with all containers `READY`. Migrations and bootstrap run inside the webapp pod rather than as separate jobs, so if the webapp stays unready, check its logs. If a pod is stuck in `Pending` or `CrashLoopBackOff`, jump to [Troubleshooting](#troubleshooting). -<Note> - GitOps tools that render with `helm template` have no cluster access, so the check is - skipped and a missing key still reaches the rollout. Verify the keys yourself before syncing. -</Note> +Once you can sign in and trigger a run, you've validated the stack. For a deployment that survives restarts, upgrades, and load, move on to the production install. -### 4.5.6: ClickHouse credentials moved +## Production install -The bundled ClickHouse password moved out of the subchart-generated Secret and into the chart-managed datastore Secret: +Running external datastores is the requirement that separates a production install from an evaluation one. You supply your own Postgres, Redis, ClickHouse, and object storage, manage secrets yourself, and point the chart at each service. The steps below layer onto the same `helm upgrade` command, driven by a custom values file. -| | Up to 4.5.5 | 4.5.6 and later | -| ------ | ---------------------- | --------------------------- | -| Secret | `<release>-clickhouse` | `trigger-datastore` | -| Key | `admin-password` | `clickhouse-admin-password` | +### Secrets -The webapp and the ClickHouse server both read the new location, so the upgrade itself needs no action. Repoint anything outside the chart that reads the old Secret — a maintenance CronJob, a Grafana datasource, an external secret sync: +Application, control-plane, and bundled-datastore secrets left unset are generated on first install and **retained across `helm upgrade`** - they are never rotated automatically, so sessions, encrypted data, and datastore volumes survive upgrades. This auto-generation is the default and needs no configuration. -```bash -kubectl get secret trigger-datastore -n trigger \ - -o jsonpath='{.data.clickhouse-admin-password}' | base64 -d -``` +For production, supply your own Secret instead so credentials live outside the release. With `secrets.existingSecret` set, the chart generates nothing and reads every key from your Secret; a preflight check fails the install - before touching a running release - and lists any keys it can't find. -The same move applies to the bundled PostgreSQL (`postgres-password`) and MinIO (`minio-root-user`, `minio-root-password`). +To pin values inline instead of letting the chart generate them - for example to share `MANAGED_WORKER_SECRET` with an external supervisor - set them under `secrets`: -### Bitnami ClickHouse to the official image +```yaml +secrets: + enabled: true + sessionSecret: "your-32-char-hex-secret-1" + magicLinkSecret: "your-32-char-hex-secret-2" + # ... +``` -The bundled ClickHouse runs the official `clickhouse/clickhouse-server` image instead of the Bitnami subchart. The chart adopts your existing data volume automatically, with three caveats covered under [ClickHouse](#clickhouse) in external services: rendering without cluster access, pinned Bitnami image tags, and storage that doesn't support `fsGroup`. +To use an existing Secret instead, leave the values unset and set `existingSecret`. It must contain at least `SESSION_SECRET`, `MAGIC_LINK_SECRET`, `ENCRYPTION_KEY`, `PROVIDER_SECRET`, `COORDINATOR_SECRET`, and `MANAGED_WORKER_SECRET` - plus `s3-auth-access-key-id` and `s3-auth-secret-access-key` if you deploy the bundled MinIO with `s3.auth.existingSecret` cleared: -<Note> - The data volume is `ReadWriteOnce`, so during the upgrade the new ClickHouse pod may sit in - `ContainerCreating` with a multi-attach warning for a minute while the old pod still holds the - volume. This clears itself once the old pod is removed later in the same upgrade - don't abort. -</Note> +```yaml +secrets: + enabled: false + existingSecret: "your-existing-secret" +``` -## Configuration +The chart wires `PROVIDER_SECRET` and `COORDINATOR_SECRET` into the webapp deployment, and the pre-upgrade check requires both, so an existing Secret must include them. For a deployment that already runs, add them before upgrading: -Most values map directly to the environment variables documented in the [webapp](/self-hosting/env/webapp) and [supervisor](/self-hosting/env/supervisor) environment variable overview. +```bash +kubectl patch secret my-trigger-secrets -n trigger --type merge -p \ + "{\"stringData\":{\"PROVIDER_SECRET\":\"$(openssl rand -hex 16)\",\"COORDINATOR_SECRET\":\"$(openssl rand -hex 16)\"}}" +``` -**Naming convention:** +There is no clean migration for a compromised `ENCRYPTION_KEY`: changing it makes existing encrypted data unreadable. If a deployment is still running a previously published default and cannot rotate yet, set `ALLOW_INSECURE_DEFAULT_SECRETS=true` on the webapp to keep booting while you plan a migration. -- Environment variables use `UPPER_SNAKE_CASE` -- Helm values use `camelCase` +<Warning> + GitOps tools that render with `helm template` (e.g. Argo CD) cannot read the existing + secret, so they regenerate auto-generated values on every sync - which rotates them. If you + deploy via GitOps, always supply your own `secrets.existingSecret` (and datastore + credentials) so nothing is generated in-cluster. +</Warning> + +### Custom values -**Example mapping:** +Most values map directly to the environment variables documented in the [webapp](/self-hosting/env/webapp) and [supervisor](/self-hosting/env/supervisor) environment variable overview. Environment variables use `UPPER_SNAKE_CASE`; Helm values use `camelCase`: ```bash # Environment variable APP_ORIGIN=https://trigger.example.com # Becomes Helm value -config: +webapp: appOrigin: "https://trigger.example.com" ``` -### Default values - -The following commands will display the default values: +Print the full default surface for the version you're deploying: ```bash # Specific version @@ -164,45 +194,14 @@ helm show values oci://ghcr.io/triggerdotdev/charts/trigger \ --version "^4.5.0" ``` -### Custom values - -The default values are insecure and are only suitable for testing. You will need to configure your own secrets as a bare minimum. - -Create a `values-custom.yaml` file to override the defaults. For example: +Create a `values-custom.yaml` that overrides the defaults with your URLs and resource requests: ```yaml -# Leave these unset to have the chart auto-generate strong values on first -# install (retained across upgrades). Set them explicitly only if you need to -# control the value - e.g. sharing MANAGED_WORKER_SECRET with an external -# supervisor - or use an existingSecret. -secrets: - enabled: true - sessionSecret: "your-32-char-hex-secret-1" - magicLinkSecret: "your-32-char-hex-secret-2" - # ... - -# Recommended: existingSecret, must contain at least the following keys: -# - SESSION_SECRET -# - MAGIC_LINK_SECRET -# - ENCRYPTION_KEY -# - PROVIDER_SECRET -# - COORDINATOR_SECRET -# - MANAGED_WORKER_SECRET -# Plus s3-auth-access-key-id and s3-auth-secret-access-key if you deploy the -# bundled MinIO with s3.auth.existingSecret cleared. The chart lists any keys -# it can't find and fails the install before touching a running release. -secrets: - enabled: false - existingSecret: "your-existing-secret" - -# Application URLs -config: +# Application URLs and resource limits +webapp: appOrigin: "https://trigger.example.com" loginOrigin: "https://trigger.example.com" apiOrigin: "https://trigger.example.com" - -# Resource limits -webapp: resources: requests: cpu: 1000m @@ -231,27 +230,7 @@ helm upgrade -n trigger --install trigger \ -f values-custom.yaml ``` -### Secret generation and rotation - -Application, control-plane, and bundled-datastore secrets left unset are generated on -first install and **retained across `helm upgrade`** - they are never rotated -automatically, so sessions, encrypted data, and datastore volumes survive upgrades. - -<Warning> - GitOps tools that render with `helm template` (e.g. Argo CD) cannot read the existing - secret, so they regenerate these values on every sync - which rotates them. If you - deploy via GitOps, always supply your own `secrets.existingSecret` (and datastore - credentials) so nothing is generated in-cluster. -</Warning> - -There is no clean migration for a compromised `ENCRYPTION_KEY`: changing it makes -existing encrypted data unreadable. If a deployment is still running a previously -published default and cannot rotate yet, set `ALLOW_INSECURE_DEFAULT_SECRETS=true` on -the webapp to keep booting while you plan a migration. - -### Extra env - -You can set extra environment variables on all services. For example: +Set extra environment variables on any service: ```yaml webapp: @@ -260,9 +239,7 @@ webapp: value: "extra-value" ``` -### Extra annotations - -You can set extra annotations on all services. For example: +Set extra annotations on any service: ```yaml webapp: @@ -272,7 +249,7 @@ webapp: ### External services -You can disable the built-in services and use external services instead. The chart supports both direct configuration and existing Kubernetes secrets for secure credential management. +Disable each built-in datastore and point the chart at your own. Every service supports direct configuration or an existing Kubernetes secret; use existing secrets to keep credentials out of `values.yaml` and Helm releases. #### PostgreSQL @@ -330,35 +307,11 @@ redis: #### ClickHouse -<Note>Trigger.dev requires ClickHouse 25.8 or newer.</Note> +Migration from the old Bitnami subchart to the official `clickhouse/clickhouse-server` image is automatic - a normal upgrade moves you across with no manual step. The one exception: if you pinned `clickhouse.image` to a Bitnami tag, switch it to an official `clickhouse/clickhouse-server` tag first, since Bitnami tags don't exist in the official repository. <Note> - When upgrading from a chart version that bundled ClickHouse via the Bitnami subchart, the chart - automatically adopts the existing data volume, so no manual migration is needed. If you render - manifests without cluster access (for example with GitOps tools that use `helm template`), set - `clickhouse.persistence.existingClaim` to the old PVC name - (`data-<release>-clickhouse-shard0-0`) to keep your data — auto-detection can't run there, and - skipping this starts ClickHouse on a fresh empty volume. If that happened, your old data is - still on the old PVC: delete the ClickHouse StatefulSet with `--cascade=orphan` (its volume - configuration is immutable), set `existingClaim`, and sync again. If you render without cluster - access, pin `clickhouse.persistence.existingClaim` for good once the volume is adopted: a later - render where the lookup can't see the PVC would otherwise try to re-add `volumeClaimTemplates`, - which the API server rejects on an existing StatefulSet. -</Note> - -<Note> - If you pinned `clickhouse.image` to a Bitnami repository or tag in your values, update it to - the official `clickhouse/clickhouse-server` image — Bitnami tags don't exist there. The - bundled ClickHouse is single-node: the old Bitnami subchart keys (`shards`, `replicaCount`, - `keeper`) are no longer supported, so use an external ClickHouse for clustered setups. -</Note> - -<Note> - When a volume is adopted from the Bitnami-based chart, a one-time init container fixes its - ownership for the non-root ClickHouse server. This runs automatically on `helm upgrade` (and - when you set `clickhouse.persistence.existingClaim`), so storage that doesn't support `fsGroup` - ownership changes (NFS, hostPath, local-path) works without manual steps. Set - `clickhouse.volumePermissions.enabled: true` to force it in other cases. + Trigger.dev requires ClickHouse 25.8 or newer. The bundled ClickHouse is single-node — run + ClickHouse externally for a clustered or replicated setup. </Note> **Direct configuration:** @@ -375,8 +328,7 @@ clickhouse: <Note> An inline external `username`/`password` is percent-encoded into the connection URL for you, so - store the **raw** value - special characters like `@ : / %` are handled automatically. If you - previously percent-encoded the password by hand to work around this, switch back to the raw value. + store the **raw** value - special characters like `@ : / %` are handled automatically. (Credentials from `existingSecret` are injected at runtime and are unaffected.) </Note> @@ -393,14 +345,15 @@ clickhouse: # existingSecretKey: "clickhouse-password" # default (optional) ``` -#### S3 Object Storage +#### Object storage + +Object storage holds deploy artifacts and large payloads. See the [Docker object storage setup](/self-hosting/docker#object-storage) for conceptual information. Disable the bundled MinIO and point at any S3-compatible endpoint. **Direct configuration:** ```yaml -minio: - deploy: false s3: + deploy: false external: endpoint: "https://s3.amazonaws.com" accessKeyId: "my-access-key" @@ -410,21 +363,19 @@ s3: **Using existing secrets (recommended):** ```yaml -minio: - deploy: false s3: + deploy: false external: endpoint: "https://s3.amazonaws.com" existingSecret: "s3-credentials" - # Optional: Use secretKeys to specify the key names in the secret - # secretKeys: - # accessKeyIdKey: "access-key-id" # default - # secretAccessKeyKey: "secret-access-key" # default + # Optional: override the key names in the secret + # existingSecretAccessKeyIdKey: "access-key-id" # default + # existingSecretSecretAccessKeyKey: "secret-access-key" # default ``` ### PostgreSQL SSL with custom CA certificates -When connecting to PostgreSQL instances that require custom CA certificates (such as AWS RDS with SSL verification), you can mount the CA certificate as a volume and configure the webapp to use it: +When connecting to PostgreSQL instances that require custom CA certificates (such as AWS RDS with SSL verification), mount the CA certificate as a volume and configure the webapp to use it: ```yaml postgres: @@ -465,99 +416,9 @@ webapp: - Compatible with secret management tools (External Secrets Operator, etc.) - Follows Kubernetes security best practices -## DNS performance - -For production clusters we recommend deploying [NodeLocal DNSCache](https://kubernetes.io/docs/tasks/administer-cluster/nodelocaldns/). DNS queries — especially to managed Postgres or Redis endpoints — can be very slow under Kubernetes' default resolver, and a node-local cache typically gives a large step change in latency and throughput across the cluster. - -The default `ndots: 5` setting also forces every cluster search domain to be tried before resolving hostnames with fewer dots (the case for most external database hosts). Lowering `ndots` to `1` on the webapp and supervisor pods avoids those extra round-trips. - -## Task events - -By default, task events (timeline, logs, spans) are stored in PostgreSQL. For production deployments we recommend storing them in ClickHouse instead, it scales to much higher volumes and avoids unbounded growth of the `TaskEvent` table. - -ClickHouse is already deployed by the chart, so no extra services are required. To enable, set `EVENT_REPOSITORY_DEFAULT_STORE` on the webapp via `extraEnvVars`: - -```yaml -webapp: - extraEnvVars: - - name: EVENT_REPOSITORY_DEFAULT_STORE - value: "clickhouse_v2" -``` - -This only affects new runs; existing runs continue to read from wherever their events were originally stored. - -## Realtime streams - -Realtime streams power AI-agent token streaming and run streams. They default to **v2**, backed by the bundled `s2` deployment — [s2-lite](https://s2.dev), the open-source, self-hostable S2 server. The chart deploys it with a persistent volume, so no extra services are required. - -To fall back to the Redis-backed **v1** streams, set the default version to `v1`: - -```yaml -s2: - defaultStreamVersion: "v1" -``` - -To use a hosted S2 at [s2.dev](https://s2.dev) instead of the bundled s2-lite, disable the bundled deployment and point at your basin. Supply the access token via an existing secret: - -```yaml -s2: - deploy: false - skipAccessTokens: false - external: - endpoint: "https://your-basin.b.aws.s2.dev/v1" - existingSecret: "s2-credentials" - existingSecretAccessTokenKey: "access-token" - basin: "your-basin" -``` - -To disable realtime streams v2 entirely and use v1, set `s2.deploy: false` with no external endpoint. See `helm show values` for all `s2` options. - -## Worker token +### Registry setup -When using the default bootstrap configuration, worker creation and authentication is handled automatically. The webapp generates a worker token and makes it available to the supervisor via a shared volume. - -### Bootstrap (default) - -```yaml -webapp: - bootstrap: - enabled: true - workerGroupName: "bootstrap" -``` - -### Manual - -If you need to set up workers separately or use a custom token: - -1. Get the worker token from the webapp logs: - -```bash -kubectl logs deployment/trigger-webapp -n trigger | grep -A15 "Worker Token" -``` - -2. Create a secret with the token: - -```bash -kubectl create secret generic worker-token \ - --from-literal=token=tr_wgt_your_token_here \ - -n trigger -``` - -3. Configure the supervisor to use the secret: - -```yaml -supervisor: - bootstrap: - enabled: false - workerToken: - secret: - name: "worker-token" - key: "token" -``` - -## Registry setup - -See the [Docker registry setup](/self-hosting/docker#registry-setup) for conceptual information. The configuration is specified in your `values.yaml`: +Built deploy images are pushed to a registry. See the [Docker registry setup](/self-hosting/docker#registry-setup) for conceptual information. Configure an external registry in your `values.yaml`: ```yaml # Use external registry (recommended) @@ -566,8 +427,7 @@ registry: # Part of deployment image ref, for example: your-registry.example.com/your-company/proj_123:20250625.1.prod repositoryNamespace: "your-company" external: - host: "your-registry.example.com" - port: 5000 + host: "your-registry.example.com:5000" auth: enabled: true username: "your-username" @@ -575,32 +435,19 @@ registry: ``` <Note> - The internal registry (`registry.external: false`) is experimental and requires proper TLS setup + The internal registry (`registry.deploy: true`) is experimental and requires proper TLS setup and additional cluster configuration. Use an external registry for production. </Note> -## Object storage - -See the [Docker object storage setup](/self-hosting/docker#object-storage) for conceptual information. The defaults will use built-in MinIO, but you can use an external S3-compatible storage. The configuration is specified in your `values.yaml`: +### DNS performance -```yaml -# Use external S3-compatible storage -minio: - deploy: false - external: - url: "https://s3.amazonaws.com" - # or: "https://your-minio.com:9000" +For production clusters we recommend deploying [NodeLocal DNSCache](https://kubernetes.io/docs/tasks/administer-cluster/nodelocaldns/). DNS queries — especially to managed Postgres or Redis endpoints — can be very slow under Kubernetes' default resolver, and a node-local cache typically gives a large step change in latency and throughput across the cluster. -# Configure credentials -secrets: - objectStore: - accessKeyId: "admin" - secretAccessKey: "very-safe-password" -``` +The default `ndots: 5` setting also forces every cluster search domain to be tried before resolving hostnames with fewer dots (the case for most external database hosts). Lowering `ndots` to `1` on the webapp and supervisor pods avoids those extra round-trips. -## Authentication +### Authentication -Authentication options are identical to the [Docker-based installation](/self-hosting/docker#authentication). The configuration is specified in your `values.yaml`: +Authentication options are identical to the [Docker-based installation](/self-hosting/docker#authentication). Configure the provider in your `values.yaml`: **GitHub OAuth:** @@ -637,7 +484,91 @@ webapp: value: "^(user1@company\\.com|user2@company\\.com)$" ``` -## Version locking +### Worker token + +The supervisor authenticates to the webapp with a worker token. The default bootstrap path generates one for you; supply your own token to manage workers separately. + +**Bootstrap (default):** the webapp generates a worker token and makes it available to the supervisor via a shared volume. + +```yaml +webapp: + bootstrap: + enabled: true + workerGroupName: "bootstrap" +``` + +**Manual:** to set up workers separately or use a custom token: + +<Steps> + <Step title="Read the worker token from the webapp logs"> + ```bash + kubectl logs deployment/trigger-webapp -n trigger | grep -A15 "Worker Token" + ``` + </Step> + + <Step title="Create a secret with the token"> + ```bash + kubectl create secret generic worker-token \ + --from-literal=token=tr_wgt_your_token_here \ + -n trigger + ``` + </Step> + + <Step title="Point the supervisor at the secret"> + ```yaml + supervisor: + bootstrap: + enabled: false + workerToken: + secret: + name: "worker-token" + key: "token" + ``` + </Step> +</Steps> + +With external datastores, secrets, registry, and authentication in place, you have a production install. If you only wanted to try Trigger.dev, the [evaluation install](#evaluation-install) gets you there in one command. + +## Operating your deployment + +### Upgrading + +`helm upgrade` is safe to run as-is. Your generated secrets, data volumes, and the bundled ClickHouse are preserved and migrated automatically, so a normal upgrade needs no preparation. A few situations need one manual step first — expand any that apply to you. + +<AccordionGroup> + <Accordion title="You supply your own secret (secrets.existingSecret)"> + With `secrets.existingSecret` set, the chart reads every key from your Secret and generates none, so any key the app needs must already be present there. + + As of chart 4.5.6 the webapp deployment references `PROVIDER_SECRET` and `COORDINATOR_SECRET`, and the pre-upgrade check requires both, so an existing Secret must carry them. Add both before upgrading: + + ```bash + kubectl patch secret my-trigger-secrets -n trigger --type merge -p \ + "{\"stringData\":{\"PROVIDER_SECRET\":\"$(openssl rand -hex 16)\",\"COORDINATOR_SECRET\":\"$(openssl rand -hex 16)\"}}" + ``` + + If a required key is missing, the upgrade stops before touching your running release and tells you which to add. + </Accordion> + + <Accordion title="Something outside the chart reads a datastore password"> + The bundled datastore passwords live in the `trigger-datastore` Secret. If a maintenance job, a dashboard, or a secret sync reads one directly, point it there: + + ```bash + kubectl get secret trigger-datastore -n trigger \ + -o jsonpath='{.data.clickhouse-admin-password}' | base64 -d + ``` + + The keys are `clickhouse-admin-password`, `postgres-password`, and `minio-root-user` / `minio-root-password`. + </Accordion> + + <Accordion title="You render manifests without cluster access (Argo CD, plain helm template)"> + During an upgrade the chart reads the cluster to keep your data and to check your Secret. Renderers that run offline can't do that, so set these yourself: + + - Point `clickhouse.persistence.existingClaim` at your ClickHouse data volume (`data-<release>-clickhouse-shard0-0`) and keep it set on future syncs, so ClickHouse reuses its data instead of starting on an empty volume. + - Confirm `PROVIDER_SECRET` and `COORDINATOR_SECRET` are already in your Secret before syncing. + </Accordion> +</AccordionGroup> + +### Version locking <Note> Trigger.dev 4.5.0 is the last version we officially support for running v3 (SDK v3) tasks. If @@ -645,7 +576,7 @@ webapp: later reject v3 triggers and deploys with an upgrade message. </Note> -You can lock versions in two ways: +Lock versions in two ways. **Helm chart version (recommended):** @@ -677,7 +608,7 @@ supervisor: The chart version's `appVersion` field determines the default image tags. Newer image tags may be incompatible with older chart versions and vice versa. -## Troubleshooting +### Troubleshooting **Check logs:** @@ -722,18 +653,58 @@ kubectl delete namespace trigger See the [Docker troubleshooting](/self-hosting/docker#troubleshooting) section for more information. -## CLI usage +### Task events + +By default, task events (timeline, logs, spans) are stored in PostgreSQL. For production deployments we recommend storing them in ClickHouse instead, it scales to much higher volumes and avoids unbounded growth of the `TaskEvent` table. + +ClickHouse is already deployed by the chart, so no extra services are required. To enable, set `EVENT_REPOSITORY_DEFAULT_STORE` on the webapp via `extraEnvVars`: + +```yaml +webapp: + extraEnvVars: + - name: EVENT_REPOSITORY_DEFAULT_STORE + value: "clickhouse_v2" +``` + +This only affects new runs; existing runs continue to read from wherever their events were originally stored. + +### Realtime streams + +Realtime streams power AI-agent token streaming and run streams. They default to **v2**, backed by the bundled `s2` deployment — [s2-lite](https://s2.dev), the open-source, self-hostable S2 server. The chart deploys it with a persistent volume, so no extra services are required. + +To fall back to the Redis-backed **v1** streams, set the default version to `v1`: + +```yaml +s2: + defaultStreamVersion: "v1" +``` + +To use a hosted S2 at [s2.dev](https://s2.dev) instead of the bundled s2-lite, disable the bundled deployment and point at your basin. Supply the access token via an existing secret: + +```yaml +s2: + deploy: false + skipAccessTokens: false + external: + endpoint: "https://your-basin.b.aws.s2.dev/v1" + existingSecret: "s2-credentials" + existingSecretAccessTokenKey: "access-token" + basin: "your-basin" +``` + +To disable realtime streams v2 entirely and use v1, set `s2.deploy: false` with no external endpoint. See `helm show values` for all `s2` options. + +### CLI usage See the [Docker CLI usage](/self-hosting/docker#cli-usage) section, the commands are identical regardless of deployment method. -## CI / GitHub Actions +### CI / GitHub Actions -When running the CLI in a CI environment, your login profiles won't be available. Instead, you can use the `TRIGGER_API_URL` and `TRIGGER_ACCESS_TOKEN` environment -variables to point at your self-hosted instance and authenticate. +When running the CLI in a CI environment, your login profiles won't be available. Instead, use the `TRIGGER_API_URL` and `TRIGGER_ACCESS_TOKEN` environment variables to point at your self-hosted instance and authenticate. For more detailed instructions, see the [GitHub Actions guide](/github-actions). -## Telemetry +### Telemetry By default, the Trigger.dev webapp sends telemetry data to our servers. This data is used to improve the product and is not shared with third parties. To disable telemetry, set in your `values.yaml`: From 763b5dc5827ae959a17f007710e08ae9c4461b8d Mon Sep 17 00:00:00 2001 From: Chris Arderne <chris@trigger.dev> Date: Mon, 3 Aug 2026 14:00:29 +0100 Subject: [PATCH 60/67] feat(webapp): enforce scopes for environment API keys (#4389) ## Summary Environment API keys backed by the additional-key table can authenticate API requests using their stored effective scopes. Revoked and expired keys are rejected, branch environments retain their existing routing behavior, and last-used timestamps are updated on a throttled best-effort basis. ## Design API route builders receive the resolved ability and reject restricted keys on routes without an authorization declaration. Existing deployment, environment variable, queue, run, task, batch, session, and waitpoint routes declare the resources they access. Trigger and batch responses return server-signed public access tokens, so additional keys never need access to the environment signing secret. Root-key rotation also keeps public tokens valid for the existing grace window. ## Feature notes - Root environment keys remain unrestricted for backward compatibility. Additional keys enforce their persisted scopes and fail closed on routes without an authorization declaration. - Machine-key requests never exchange one credential for another. Additional keys cannot retrieve the root key, and rotated root keys are not upgraded during their grace window. - Public JWT validation remains host-owned, while installed RBAC plugins continue to supply root-key abilities. - Unfiltered session and run listings preserve existing broad task-read behavior. Filtered requests enforce the supplied task identifiers. - Related-run summaries remain embedded in run retrieval for API compatibility. Retrieving or mutating a related run independently still requires permission for that run. - Queue management authorizes at collection scope, matching the queue permissions currently issued. - Batch responses deliberately include server-signed public access tokens for all clients. Selected-task credentials continue using their original credential for per-item authorization. - Two-phase batches authorize declared task identifiers before creation and authorize every streamed item. Streaming paths that cannot declare the complete task set remain fail closed. - Authentication telemetry records successful credential resolution separately from subsequent resource-authorization failures. - API keys are high-entropy random tokens. SHA-256 is intentionally used for deterministic indexed lookup, not password hashing. ## Deployment notes The schema migration must be present before this code is deployed. Because bearer resolution runs on every authenticated request, deploy the resolver with additional-key lookup disabled, verify root-key and public-token parity, then enable lookup before any additional keys can be issued. The multi-task authorization tightening changes the result for narrowly scoped tokens that request tasks outside their grants. Observe would-deny results before enforcing that check. Request-idempotency keys are also newly isolated by environment and task, so a retry crossing the deployment boundary may execute once more before old cache entries expire. ## Follow-ups - [x] Add a system-wide kill switch for additional-key lookup, defaulted off for the initial deployment. - [x] Add authentication observability by credential kind, result, latency, and lookup path without recording credential values. - [ ] ~Add would-deny observability and an independent enforcement switch for multi-task authorization.~ - [ ] ~Add an independent switch for server-issued batch tokens while root-key parity is verified.~ - [ ] Confirm every API route reachable by a restricted key has an explicit authorization declaration or intentionally fails closed. - [x] Verify root-key rotation, revoked-key grace, and public-token validation through each bearer resolver path. --- .changeset/friendly-keys-batch-tokens.md | 6 + .../multi-task-scope-enforcement.md | 6 + .../scoped-environment-api-key-auth.md | 6 + .../app/models/runtimeEnvironment.server.ts | 179 ++++++-- .../v3/ApiRetrieveRunPresenter.server.ts | 6 +- apps/webapp/app/routes/api.v1.artifacts.ts | 16 +- ...yments.$deploymentId.background-workers.ts | 13 +- ...api.v1.deployments.$deploymentId.cancel.ts | 15 +- .../api.v1.deployments.$deploymentId.fail.ts | 13 +- ...i.v1.deployments.$deploymentId.finalize.ts | 13 +- ...loymentId.generate-registry-credentials.ts | 15 +- ...i.v1.deployments.$deploymentId.progress.ts | 15 +- .../api.v1.deployments.$deploymentId.ts | 13 +- ....deployments.$deploymentVersion.promote.ts | 13 +- .../app/routes/api.v1.deployments.latest.ts | 13 +- apps/webapp/app/routes/api.v1.deployments.ts | 13 +- .../api.v1.projects.$projectRef.$env.ts | 37 +- ...rojects.$projectRef.envvars.$slug.$name.ts | 38 +- ...ojects.$projectRef.envvars.$slug.import.ts | 22 +- ...i.v1.projects.$projectRef.envvars.$slug.ts | 38 +- .../api.v1.projects.$projectRef.envvars.ts | 15 +- ...queues.$queueParam.concurrency.override.ts | 4 + ...v1.queues.$queueParam.concurrency.reset.ts | 4 + .../routes/api.v1.queues.$queueParam.pause.ts | 4 + .../app/routes/api.v1.queues.$queueParam.ts | 4 + apps/webapp/app/routes/api.v1.queues.ts | 1 + .../routes/api.v1.runs.$runParam.replay.ts | 198 ++++---- apps/webapp/app/routes/api.v1.runs.ts | 29 +- apps/webapp/app/routes/api.v1.sessions.ts | 19 +- .../app/routes/api.v1.tasks.$taskId.batch.ts | 204 ++++----- .../routes/api.v1.tasks.$taskId.trigger.ts | 23 +- apps/webapp/app/routes/api.v1.tasks.batch.ts | 50 +- .../app/routes/api.v1.waitpoints.tokens.ts | 24 +- ...i.v2.deployments.$deploymentId.finalize.ts | 13 +- .../routes/api.v2.runs.$runParam.cancel.ts | 16 +- apps/webapp/app/routes/api.v2.tasks.batch.ts | 65 +-- .../routes/api.v3.batches.$batchId.items.ts | 39 +- apps/webapp/app/routes/api.v3.batches.ts | 97 ++-- ...i.v3.deployments.$deploymentId.finalize.ts | 13 +- apps/webapp/app/services/apiAuth.server.ts | 99 +++- .../services/authFeatureControls.server.ts | 13 + .../app/services/authFeatureControls.ts | 13 + .../app/services/authTelemetry.server.ts | 168 +++++++ .../environmentVariableApiAccess.server.ts | 89 +++- .../publicAccessTokenResponse.server.ts | 33 ++ apps/webapp/app/services/rbac.server.ts | 2 + .../app/services/realtime/jwtAuth.server.ts | 72 +-- .../routeBuilders/apiBuilder.server.ts | 94 +++- .../app/utils/batchItemAuthorization.ts | 96 ++++ .../utils/parentRunAuthorization.server.ts | 27 ++ .../app/utils/parentRunAuthorization.ts | 13 + .../app/utils/requestIdempotency.test.ts | 22 + .../webapp/app/utils/requestIdempotencyKey.ts | 12 + apps/webapp/app/v3/featureFlags.ts | 9 + .../mollifier/resolveRunForMutation.server.ts | 37 +- apps/webapp/test/apiAuthScope.test.ts | 163 +++++++ .../test/apiBuilderAuthorization.test.ts | 71 +++ .../apiRetrieveRunPresenter.readroute.test.ts | 5 + apps/webapp/test/auth-api.e2e.full.test.ts | 95 +++- apps/webapp/test/authFeatureControls.test.ts | 19 + .../test/environmentVariableApiAccess.test.ts | 111 +++++ .../test/findEnvironmentByApiKey.test.ts | 207 ++++++++- .../mollifierResolveRunForMutation.test.ts | 20 +- .../projectEnvironmentCredentialRoute.test.ts | 127 +++++ .../test/publicAccessTokenResponse.test.ts | 49 ++ apps/webapp/test/rbacFallbackBranch.test.ts | 413 ++++++++++++++++- .../streamBatchItemsAuthorization.test.ts | 136 ++++++ .../rbac/src/apiKeyPolicies.test.ts | 325 +++++++++++++ .../rbac/src/bearerCredentials.ts | 433 ++++++++++++++++++ internal-packages/rbac/src/fallback.ts | 220 +-------- internal-packages/rbac/src/index.ts | 106 ++++- packages/core/package.json | 15 + packages/core/src/v3/schemas/api.ts | 2 + .../src/v3/schemas/idempotencyKey.test.ts | 18 + packages/trigger-sdk/src/v3/shared.test.ts | 18 +- packages/trigger-sdk/src/v3/shared.ts | 5 + 76 files changed, 3819 insertions(+), 850 deletions(-) create mode 100644 .changeset/friendly-keys-batch-tokens.md create mode 100644 .server-changes/multi-task-scope-enforcement.md create mode 100644 .server-changes/scoped-environment-api-key-auth.md create mode 100644 apps/webapp/app/services/authFeatureControls.server.ts create mode 100644 apps/webapp/app/services/authFeatureControls.ts create mode 100644 apps/webapp/app/services/authTelemetry.server.ts create mode 100644 apps/webapp/app/services/publicAccessTokenResponse.server.ts create mode 100644 apps/webapp/app/utils/batchItemAuthorization.ts create mode 100644 apps/webapp/app/utils/parentRunAuthorization.server.ts create mode 100644 apps/webapp/app/utils/parentRunAuthorization.ts create mode 100644 apps/webapp/app/utils/requestIdempotency.test.ts create mode 100644 apps/webapp/app/utils/requestIdempotencyKey.ts create mode 100644 apps/webapp/test/apiAuthScope.test.ts create mode 100644 apps/webapp/test/apiBuilderAuthorization.test.ts create mode 100644 apps/webapp/test/authFeatureControls.test.ts create mode 100644 apps/webapp/test/environmentVariableApiAccess.test.ts create mode 100644 apps/webapp/test/projectEnvironmentCredentialRoute.test.ts create mode 100644 apps/webapp/test/publicAccessTokenResponse.test.ts create mode 100644 apps/webapp/test/streamBatchItemsAuthorization.test.ts create mode 100644 internal-packages/rbac/src/apiKeyPolicies.test.ts create mode 100644 internal-packages/rbac/src/bearerCredentials.ts diff --git a/.changeset/friendly-keys-batch-tokens.md b/.changeset/friendly-keys-batch-tokens.md new file mode 100644 index 00000000000..90506d48d28 --- /dev/null +++ b/.changeset/friendly-keys-batch-tokens.md @@ -0,0 +1,6 @@ +--- +"@trigger.dev/core": patch +"@trigger.dev/sdk": patch +--- + +Allow task-scoped environment API keys to run batch operations for their permitted tasks. The SDK declares the batch's task set before creation, and `@trigger.dev/core/v3/apiKeys` now exports the additional-key format helper. diff --git a/.server-changes/multi-task-scope-enforcement.md b/.server-changes/multi-task-scope-enforcement.md new file mode 100644 index 00000000000..27e31cf5ec4 --- /dev/null +++ b/.server-changes/multi-task-scope-enforcement.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Requests spanning multiple tasks now require permission for every requested task instead of accepting permission for only one task. diff --git a/.server-changes/scoped-environment-api-key-auth.md b/.server-changes/scoped-environment-api-key-auth.md new file mode 100644 index 00000000000..948f44d1977 --- /dev/null +++ b/.server-changes/scoped-environment-api-key-auth.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: feature +--- + +Additional environment API keys can authenticate API requests using their configured permissions, with revoked and expired keys rejected. Batch responses use server-issued public access tokens so additional keys never need the environment signing secret. diff --git a/apps/webapp/app/models/runtimeEnvironment.server.ts b/apps/webapp/app/models/runtimeEnvironment.server.ts index 939cd55e94d..84d2979bb10 100644 --- a/apps/webapp/app/models/runtimeEnvironment.server.ts +++ b/apps/webapp/app/models/runtimeEnvironment.server.ts @@ -5,7 +5,11 @@ import { runStore } from "~/v3/runStore.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; import { logger } from "~/services/logger.server"; import { getUsername } from "~/utils/username"; +import { hashApiKey } from "~/utils/apiKeys"; +import { isAdditionalApiKey } from "@trigger.dev/core/v3/apiKeys"; import { isDefaultDevBranch, sanitizeBranchName } from "@trigger.dev/core/v3/utils/gitBranch"; +import { scopesGrantFullAccess } from "@trigger.dev/rbac"; +import { authFeatureControls } from "~/services/authFeatureControls.server"; export type { RuntimeEnvironment }; @@ -93,11 +97,22 @@ export function toAuthenticated( }; } -export async function findEnvironmentByApiKey( +export type ApiKeyEnvironmentResolution = + | { ok: true; environment: AuthenticatedEnvironment } + | { ok: false; reason: "not-found" | "restricted" | "disabled" }; + +/** + * Resolve an environment from a raw API key for legacy routes that do not + * declare authorization. Additional keys are accepted only when their stored + * scopes explicitly grant full access; restricted keys fail closed here + * (`reason: "restricted"`, so callers can explain the rejection). + */ +async function resolveEnvironmentByApiKey( apiKey: string, branchName: string | undefined, - tx: PrismaClientOrTransaction = $replica -): Promise<AuthenticatedEnvironment | null> { + tx: PrismaClientOrTransaction, + additionalApiKeyLookupEnabled: () => boolean +): Promise<ApiKeyEnvironmentResolution> { const branch = sanitizeBranchName(branchName) ?? undefined; const include = { @@ -112,35 +127,94 @@ export async function findEnvironmentByApiKey( : undefined, } satisfies Prisma.RuntimeEnvironmentInclude; - let environment = await tx.runtimeEnvironment.findFirst({ - where: { - apiKey, - }, - include, - }); + const now = new Date(); + const routesToAdditionalKey = isAdditionalApiKey(apiKey); + if (routesToAdditionalKey && !additionalApiKeyLookupEnabled()) { + return { ok: false, reason: "disabled" }; + } - // Fall back to keys that were revoked within the grace window - if (!environment) { + let rootEnvironment = routesToAdditionalKey + ? null + : await tx.runtimeEnvironment.findFirst({ + where: { + apiKey, + }, + include, + }); + + // Fall back to root keys that were rotated within the grace window. + if (!routesToAdditionalKey && !rootEnvironment) { const revokedApiKey = await tx.revokedApiKey.findFirst({ where: { apiKey, - expiresAt: { gt: new Date() }, + expiresAt: { gt: now }, }, include: { runtimeEnvironment: { include }, }, }); - environment = revokedApiKey?.runtimeEnvironment ?? null; + rootEnvironment = revokedApiKey?.runtimeEnvironment ?? null; } + // Additional keys are host-owned credentials. Legacy routes cannot apply a + // scoped ability, so only an explicit full-access scope is accepted. + const match = routesToAdditionalKey + ? await tx.apiKey.findFirst({ + where: { + keyHash: hashApiKey(apiKey), + revokedAt: null, + OR: [{ expiresAt: null }, { expiresAt: { gt: now } }], + }, + select: { + id: true, + lastUsedAt: true, + scopes: true, + runtimeEnvironment: { include }, + }, + }) + : null; + + if (match && !scopesGrantFullAccess(match.scopes)) { + return { ok: false, reason: "restricted" }; + } + + const additionalApiKey = match ? { id: match.id, lastUsedAt: match.lastUsedAt } : null; + let environment = rootEnvironment ?? match?.runtimeEnvironment ?? null; + if (!environment) { - return null; + return { ok: false, reason: "not-found" }; + } + + if ( + additionalApiKey && + (!additionalApiKey.lastUsedAt || + additionalApiKey.lastUsedAt < new Date(now.getTime() - 300_000)) + ) { + try { + // Deliberately the primary `prisma`, not `tx`: `tx` defaults to the + // read replica (and may be a caller's transaction), and this last-used + // telemetry write must hit the writer. It's throttled to once every 5 + // minutes per key and best-effort — auth never fails if it can't record. + await prisma.apiKey.updateMany({ + where: { + id: additionalApiKey.id, + revokedAt: null, + OR: [{ expiresAt: null }, { expiresAt: { gt: now } }], + }, + data: { lastUsedAt: now }, + }); + } catch (error) { + logger.warn("Failed to update API key last-used timestamp", { + apiKeyId: additionalApiKey.id, + error, + }); + } } //don't return deleted projects if (environment.project.deletedAt !== null) { - return null; + return { ok: false, reason: "not-found" }; } if (environment.type === "PREVIEW") { @@ -148,23 +222,26 @@ export async function findEnvironmentByApiKey( logger.warn("findEnvironmentByApiKey(): Preview env with no branch name provided", { environmentId: environment.id, }); - return null; + return { ok: false, reason: "not-found" }; } const childEnvironment = environment.childEnvironments.at(0); if (childEnvironment) { - return toAuthenticated({ - ...childEnvironment, - apiKey: environment.apiKey, - orgMember: environment.orgMember, - organization: environment.organization, - project: environment.project, - }); + return { + ok: true, + environment: toAuthenticated({ + ...childEnvironment, + apiKey: environment.apiKey, + orgMember: environment.orgMember, + organization: environment.organization, + project: environment.project, + }), + }; } //A branch was specified but no child environment was found - return null; + return { ok: false, reason: "not-found" }; } // If there is a named DEV branch (other than default), return it @@ -172,20 +249,56 @@ export async function findEnvironmentByApiKey( const childEnvironment = environment.childEnvironments.at(0); if (childEnvironment) { - return toAuthenticated({ - ...childEnvironment, - apiKey: environment.apiKey, - orgMember: environment.orgMember, - organization: environment.organization, - project: environment.project, - }); + return { + ok: true, + environment: toAuthenticated({ + ...childEnvironment, + apiKey: environment.apiKey, + orgMember: environment.orgMember, + organization: environment.organization, + project: environment.project, + }), + }; } //A branch was specified but no child environment was found - return null; + return { ok: false, reason: "not-found" }; } - return toAuthenticated(environment); + return { ok: true, environment: toAuthenticated(environment) }; +} + +/** + * Resolve an environment from a raw API key. Root and grace-window keys keep + * their legacy behavior; additional keys with restricted scopes fail closed. + */ +export async function findEnvironmentByApiKey( + apiKey: string, + branchName: string | undefined, + tx: PrismaClientOrTransaction = $replica, + additionalApiKeyLookupEnabled = authFeatureControls.additionalApiKeyLookupEnabled +): Promise<AuthenticatedEnvironment | null> { + const resolution = await resolveEnvironmentByApiKey( + apiKey, + branchName, + tx, + additionalApiKeyLookupEnabled + ); + return resolution.ok ? resolution.environment : null; +} + +/** + * Like `findEnvironmentByApiKey`, but distinguishes a restricted additional + * key (fails closed on legacy routes) from an unknown key so callers can + * return an accurate error message. + */ +export async function findEnvironmentByApiKeyWithResolution( + apiKey: string, + branchName: string | undefined, + tx: PrismaClientOrTransaction = $replica, + additionalApiKeyLookupEnabled = authFeatureControls.additionalApiKeyLookupEnabled +): Promise<ApiKeyEnvironmentResolution> { + return resolveEnvironmentByApiKey(apiKey, branchName, tx, additionalApiKeyLookupEnabled); } /** diff --git a/apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts b/apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts index c6a14a44921..075673e96d0 100644 --- a/apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts @@ -289,6 +289,8 @@ export class ApiRetrieveRunPresenter { attemptCount: taskRun.engine === "V1" ? taskRun.attempts.length : (taskRun.attemptNumber ?? 0), attempts: [], + // Related runs are an embedded projection of the authorized run, not independent reads. + // Preserve the established response shape for run-scoped credentials. relatedRuns: { root: taskRun.rootTaskRun ? await createCommonRunStructure(taskRun.rootTaskRun, this.apiVersion) @@ -297,7 +299,9 @@ export class ApiRetrieveRunPresenter { ? await createCommonRunStructure(taskRun.parentTaskRun, this.apiVersion) : undefined, children: await Promise.all( - taskRun.childRuns.map(async (r) => await createCommonRunStructure(r, this.apiVersion)) + taskRun.childRuns.map( + async (run) => await createCommonRunStructure(run, this.apiVersion) + ) ), }, }; diff --git a/apps/webapp/app/routes/api.v1.artifacts.ts b/apps/webapp/app/routes/api.v1.artifacts.ts index c74c66a222d..a706f9e04ef 100644 --- a/apps/webapp/app/routes/api.v1.artifacts.ts +++ b/apps/webapp/app/routes/api.v1.artifacts.ts @@ -4,7 +4,7 @@ import { CreateArtifactRequestBody, tryCatch, } from "@trigger.dev/core/v3"; -import { authenticateRequest } from "~/services/apiAuth.server"; +import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { ArtifactsService } from "~/v3/services/artifacts.server"; @@ -14,17 +14,19 @@ export async function action({ request }: ActionFunctionArgs) { } try { - const authenticationResult = await authenticateRequest(request, { - apiKey: true, - organizationAccessToken: false, - personalAccessToken: false, + // Artifact uploads are part of the deploy flow (deployment context archive). + const authResult = await authenticateApiKeyWithScope(request, { + action: "write", + resource: { type: "deployments" }, }); - if (!authenticationResult || !authenticationResult.result.ok) { + if (!authResult.ok) { logger.info("Invalid or missing api key", { url: request.url }); - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + return json({ error: authResult.error }, { status: authResult.status }); } + const authenticationResult = { result: authResult.authentication }; + const [, rawBody] = await tryCatch(request.json()); const body = CreateArtifactRequestBody.safeParse(rawBody ?? {}); diff --git a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.background-workers.ts b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.background-workers.ts index 9e7052b7275..f0d419ca183 100644 --- a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.background-workers.ts +++ b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.background-workers.ts @@ -2,7 +2,7 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { CreateBackgroundWorkerRequestBody } from "@trigger.dev/core/v3"; import { z } from "zod"; -import { authenticateApiRequest } from "~/services/apiAuth.server"; +import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { ServiceValidationError } from "~/v3/services/baseService.server"; import { CreateDeclarativeScheduleError } from "~/v3/services/createBackgroundWorker.server"; @@ -26,13 +26,18 @@ export async function action({ request, params }: ActionFunctionArgs) { try { // Next authenticate the request - const authenticationResult = await authenticateApiRequest(request); + const authResult = await authenticateApiKeyWithScope(request, { + action: "write", + resource: { type: "deployments" }, + }); - if (!authenticationResult) { + if (!authResult.ok) { logger.info("Invalid or missing api key", { url: request.url }); - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + return json({ error: authResult.error }, { status: authResult.status }); } + const authenticationResult = authResult.authentication; + const authenticatedEnv = authenticationResult.environment; const { deploymentId } = parsedParams.data; diff --git a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts index d14f7a8bc22..168de476758 100644 --- a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts +++ b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts @@ -1,7 +1,7 @@ import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; import { CancelDeploymentRequestBody, tryCatch } from "@trigger.dev/core/v3"; import { z } from "zod"; -import { authenticateRequest } from "~/services/apiAuth.server"; +import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { DeploymentService } from "~/v3/services/deployment.server"; @@ -21,18 +21,17 @@ export async function action({ request, params }: ActionFunctionArgs) { } try { - const authenticationResult = await authenticateRequest(request, { - apiKey: true, - organizationAccessToken: false, - personalAccessToken: false, + const authResult = await authenticateApiKeyWithScope(request, { + action: "write", + resource: { type: "deployments" }, }); - if (!authenticationResult || !authenticationResult.result.ok) { + if (!authResult.ok) { logger.info("Invalid or missing api key", { url: request.url }); - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + return json({ error: authResult.error }, { status: authResult.status }); } - const { environment: authenticatedEnv } = authenticationResult.result; + const { environment: authenticatedEnv } = authResult.authentication; const { deploymentId } = parsedParams.data; const [, rawBody] = await tryCatch(request.json()); diff --git a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.fail.ts b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.fail.ts index 7ba5dd3700d..df596716a8f 100644 --- a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.fail.ts +++ b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.fail.ts @@ -2,7 +2,7 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { FailDeploymentRequestBody } from "@trigger.dev/core/v3"; import { z } from "zod"; -import { authenticateApiRequest } from "~/services/apiAuth.server"; +import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { FailDeploymentService } from "~/v3/services/failDeployment.server"; @@ -24,13 +24,18 @@ export async function action({ request, params }: ActionFunctionArgs) { try { // Next authenticate the request - const authenticationResult = await authenticateApiRequest(request); + const authResult = await authenticateApiKeyWithScope(request, { + action: "write", + resource: { type: "deployments" }, + }); - if (!authenticationResult) { + if (!authResult.ok) { logger.info("Invalid or missing api key", { url: request.url }); - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + return json({ error: authResult.error }, { status: authResult.status }); } + const authenticationResult = authResult.authentication; + const authenticatedEnv = authenticationResult.environment; const { deploymentId } = parsedParams.data; diff --git a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.finalize.ts b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.finalize.ts index f576f6eef38..7b9c5419491 100644 --- a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.finalize.ts +++ b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.finalize.ts @@ -2,7 +2,7 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { FinalizeDeploymentRequestBody } from "@trigger.dev/core/v3"; import { z } from "zod"; -import { authenticateApiRequest } from "~/services/apiAuth.server"; +import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { ServiceValidationError } from "~/v3/services/baseService.server"; import { FinalizeDeploymentService } from "~/v3/services/finalizeDeployment.server"; @@ -25,13 +25,18 @@ export async function action({ request, params }: ActionFunctionArgs) { try { // Next authenticate the request - const authenticationResult = await authenticateApiRequest(request); + const authResult = await authenticateApiKeyWithScope(request, { + action: "write", + resource: { type: "deployments" }, + }); - if (!authenticationResult) { + if (!authResult.ok) { logger.info("Invalid or missing api key", { url: request.url }); - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + return json({ error: authResult.error }, { status: authResult.status }); } + const authenticationResult = authResult.authentication; + const authenticatedEnv = authenticationResult.environment; const { deploymentId } = parsedParams.data; diff --git a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.generate-registry-credentials.ts b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.generate-registry-credentials.ts index 78488bfeeb5..fe60b9ace45 100644 --- a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.generate-registry-credentials.ts +++ b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.generate-registry-credentials.ts @@ -5,7 +5,7 @@ import { tryCatch, } from "@trigger.dev/core/v3"; import { z } from "zod"; -import { authenticateRequest } from "~/services/apiAuth.server"; +import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { DeploymentService } from "~/v3/services/deployment.server"; @@ -25,18 +25,17 @@ export async function action({ request, params }: ActionFunctionArgs) { } try { - const authenticationResult = await authenticateRequest(request, { - apiKey: true, - organizationAccessToken: false, - personalAccessToken: false, + const authResult = await authenticateApiKeyWithScope(request, { + action: "write", + resource: { type: "deployments" }, }); - if (!authenticationResult || !authenticationResult.result.ok) { + if (!authResult.ok) { logger.info("Invalid or missing api key", { url: request.url }); - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + return json({ error: authResult.error }, { status: authResult.status }); } - const { environment: authenticatedEnv } = authenticationResult.result; + const { environment: authenticatedEnv } = authResult.authentication; const { deploymentId } = parsedParams.data; const [, rawBody] = await tryCatch(request.json()); diff --git a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.progress.ts b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.progress.ts index 671f42606a2..839e5a5cf5b 100644 --- a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.progress.ts +++ b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.progress.ts @@ -1,7 +1,7 @@ import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; import { ProgressDeploymentRequestBody, tryCatch } from "@trigger.dev/core/v3"; import { z } from "zod"; -import { authenticateRequest } from "~/services/apiAuth.server"; +import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { DeploymentService } from "~/v3/services/deployment.server"; @@ -21,18 +21,17 @@ export async function action({ request, params }: ActionFunctionArgs) { } try { - const authenticationResult = await authenticateRequest(request, { - apiKey: true, - organizationAccessToken: false, - personalAccessToken: false, + const authResult = await authenticateApiKeyWithScope(request, { + action: "write", + resource: { type: "deployments" }, }); - if (!authenticationResult || !authenticationResult.result.ok) { + if (!authResult.ok) { logger.info("Invalid or missing api key", { url: request.url }); - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + return json({ error: authResult.error }, { status: authResult.status }); } - const { environment: authenticatedEnv } = authenticationResult.result; + const { environment: authenticatedEnv } = authResult.authentication; const { deploymentId } = parsedParams.data; const [, rawBody] = await tryCatch(request.json()); diff --git a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts index dd16fabf24c..6b7accd0291 100644 --- a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts +++ b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts @@ -2,7 +2,7 @@ import { type LoaderFunctionArgs, json } from "@remix-run/server-runtime"; import { type GetDeploymentResponseBody } from "@trigger.dev/core/v3"; import { z } from "zod"; import { prisma } from "~/db.server"; -import { authenticateApiRequest } from "~/services/apiAuth.server"; +import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; const ParamsSchema = z.object({ @@ -18,13 +18,18 @@ export async function loader({ request, params }: LoaderFunctionArgs) { try { // Next authenticate the request - const authenticationResult = await authenticateApiRequest(request); + const authResult = await authenticateApiKeyWithScope(request, { + action: "read", + resource: { type: "deployments" }, + }); - if (!authenticationResult) { + if (!authResult.ok) { logger.info("Invalid or missing api key", { url: request.url }); - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + return json({ error: authResult.error }, { status: authResult.status }); } + const authenticationResult = authResult.authentication; + const authenticatedEnv = authenticationResult.environment; const { deploymentId } = parsedParams.data; diff --git a/apps/webapp/app/routes/api.v1.deployments.$deploymentVersion.promote.ts b/apps/webapp/app/routes/api.v1.deployments.$deploymentVersion.promote.ts index f2c69078b9e..c7cce0d1ed3 100644 --- a/apps/webapp/app/routes/api.v1.deployments.$deploymentVersion.promote.ts +++ b/apps/webapp/app/routes/api.v1.deployments.$deploymentVersion.promote.ts @@ -2,7 +2,7 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; -import { authenticateApiRequest } from "~/services/apiAuth.server"; +import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { ServiceValidationError } from "~/v3/services/baseService.server"; import { ChangeCurrentDeploymentService } from "~/v3/services/changeCurrentDeployment.server"; @@ -25,13 +25,18 @@ export async function action({ request, params }: ActionFunctionArgs) { try { // Next authenticate the request - const authenticationResult = await authenticateApiRequest(request); + const authResult = await authenticateApiKeyWithScope(request, { + action: "write", + resource: { type: "deployments" }, + }); - if (!authenticationResult) { + if (!authResult.ok) { logger.info("Invalid or missing api key", { url: request.url }); - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + return json({ error: authResult.error }, { status: authResult.status }); } + const authenticationResult = authResult.authentication; + const authenticatedEnv = authenticationResult.environment; const url = new URL(request.url); diff --git a/apps/webapp/app/routes/api.v1.deployments.latest.ts b/apps/webapp/app/routes/api.v1.deployments.latest.ts index 7609f874255..9354db4de33 100644 --- a/apps/webapp/app/routes/api.v1.deployments.latest.ts +++ b/apps/webapp/app/routes/api.v1.deployments.latest.ts @@ -2,19 +2,24 @@ import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { WorkerInstanceGroupType } from "@trigger.dev/database"; import { prisma } from "~/db.server"; -import { authenticateApiRequest } from "~/services/apiAuth.server"; +import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; export async function loader({ request }: LoaderFunctionArgs) { try { // Next authenticate the request - const authenticationResult = await authenticateApiRequest(request); + const authResult = await authenticateApiKeyWithScope(request, { + action: "read", + resource: { type: "deployments" }, + }); - if (!authenticationResult) { + if (!authResult.ok) { logger.info("Invalid or missing api key", { url: request.url }); - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + return json({ error: authResult.error }, { status: authResult.status }); } + const authenticationResult = authResult.authentication; + const authenticatedEnv = authenticationResult.environment; const deployment = await prisma.workerDeployment.findFirst({ diff --git a/apps/webapp/app/routes/api.v1.deployments.ts b/apps/webapp/app/routes/api.v1.deployments.ts index 39988e1d13d..273259d87df 100644 --- a/apps/webapp/app/routes/api.v1.deployments.ts +++ b/apps/webapp/app/routes/api.v1.deployments.ts @@ -5,7 +5,7 @@ import { type InitializeDeploymentResponseBody, } from "@trigger.dev/core/v3"; import { $replica } from "~/db.server"; -import { authenticateApiRequest } from "~/services/apiAuth.server"; +import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; import { ServiceValidationError } from "~/v3/services/baseService.server"; @@ -18,13 +18,18 @@ export async function action({ request, params }: ActionFunctionArgs) { } // Next authenticate the request - const authenticationResult = await authenticateApiRequest(request); + const authResult = await authenticateApiKeyWithScope(request, { + action: "write", + resource: { type: "deployments" }, + }); - if (!authenticationResult) { + if (!authResult.ok) { logger.info("Invalid or missing api key", { url: request.url }); - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + return json({ error: authResult.error }, { status: authResult.status }); } + const authenticationResult = authResult.authentication; + const rawBody = await request.json(); const body = InitializeDeploymentRequestBody.safeParse(rawBody); diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.$env.ts b/apps/webapp/app/routes/api.v1.projects.$projectRef.$env.ts index 42ef412ec19..5c01c375601 100644 --- a/apps/webapp/app/routes/api.v1.projects.$projectRef.$env.ts +++ b/apps/webapp/app/routes/api.v1.projects.$projectRef.$env.ts @@ -4,11 +4,14 @@ import { z } from "zod"; import { env as processEnv } from "~/env.server"; import { authenticatedEnvironmentForAuthentication, - authenticateRequest, branchNameFromRequest, } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; -import { authorizePatEnvironmentAccess } from "~/services/environmentVariableApiAccess.server"; +import { + authenticateEnvironmentScopedApiRequest, + authorizePatEnvironmentAccess, + presentedApiKeyFromAuthentication, +} from "~/services/environmentVariableApiAccess.server"; const ParamsSchema = z.object({ projectRef: z.string(), @@ -27,15 +30,13 @@ export async function loader({ request, params }: LoaderFunctionArgs) { const { projectRef, env } = parsedParams.data; try { - const authenticationResult = await authenticateRequest(request, { - personalAccessToken: true, - organizationAccessToken: true, - apiKey: true, - }); - - if (!authenticationResult) { - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + // PAT/OAT authenticate on the legacy path; machine API keys go through + // the RBAC controller so additional keys (and their grants) are enforced. + const authResult = await authenticateEnvironmentScopedApiRequest(request, "read", "apiKeys"); + if (!authResult.ok) { + return json({ error: authResult.error }, { status: authResult.status }); } + const authenticationResult = authResult.authentication; const environment = await authenticatedEnvironmentForAuthentication( authenticationResult, @@ -44,12 +45,16 @@ export async function loader({ request, params }: LoaderFunctionArgs) { branchNameFromRequest(request) ); - // This endpoint hands the caller the environment's secret key. For a PAT - // (a user), gate it on env-tier read:apiKeys — so a restricted role can't - // pull deployed credentials (and therefore can't deploy) via the CLI. + // User tokens bootstrap the environment's secret key, so gate them on + // env-tier read:apiKeys. Machine credentials are checked against the same + // permission before their presented key is returned below. const denied = await authorizePatEnvironmentAccess({ request, authType: authenticationResult.type, + ability: + authenticationResult.type === "apiKey" && authenticationResult.result.ok + ? authenticationResult.result.ability + : undefined, organizationId: environment.organizationId, projectId: environment.project.id, envType: environment.type, @@ -58,8 +63,12 @@ export async function loader({ request, params }: LoaderFunctionArgs) { }); if (denied) return denied; + // API-key callers already possess a valid environment credential. Reuse + // exactly what they presented instead of exchanging it for the root key. + const presentedApiKey = presentedApiKeyFromAuthentication(authenticationResult); + const result: GetProjectEnvResponse = { - apiKey: environment.apiKey, + apiKey: presentedApiKey ?? environment.apiKey, name: environment.project.name, apiUrl: processEnv.API_ORIGIN ?? processEnv.APP_ORIGIN, projectId: environment.project.id, diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.$name.ts b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.$name.ts index 4e0fca9a480..49ce15ae188 100644 --- a/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.$name.ts +++ b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.$name.ts @@ -4,12 +4,14 @@ import { UpdateEnvironmentVariableRequestBody } from "@trigger.dev/core/v3"; import { z } from "zod"; import { prisma } from "~/db.server"; import { - authenticateRequest, authenticatedEnvironmentForAuthentication, branchNameFromRequest, } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; -import { authorizeEnvVarApiRequest } from "~/services/environmentVariableApiAccess.server"; +import { + authenticateEnvVarApiRequest, + authorizeEnvVarApiRequest, +} from "~/services/environmentVariableApiAccess.server"; import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; const ParamsSchema = z.object({ @@ -26,15 +28,11 @@ export async function action({ params, request }: ActionFunctionArgs) { } try { - const authenticationResult = await authenticateRequest(request, { - personalAccessToken: true, - organizationAccessToken: true, - apiKey: true, - }); - - if (!authenticationResult) { - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + const authResult = await authenticateEnvVarApiRequest(request, "write"); + if (!authResult.ok) { + return json({ error: authResult.error }, { status: authResult.status }); } + const authenticationResult = authResult.authentication; const environment = await authenticatedEnvironmentForAuthentication( authenticationResult, @@ -46,6 +44,10 @@ export async function action({ params, request }: ActionFunctionArgs) { const denied = await authorizeEnvVarApiRequest({ request, authType: authenticationResult.type, + ability: + authenticationResult.type === "apiKey" && authenticationResult.result.ok + ? authenticationResult.result.ability + : undefined, organizationId: environment.organizationId, projectId: environment.project.id, envType: environment.type, @@ -126,15 +128,11 @@ export async function loader({ params, request }: LoaderFunctionArgs) { } try { - const authenticationResult = await authenticateRequest(request, { - personalAccessToken: true, - organizationAccessToken: true, - apiKey: true, - }); - - if (!authenticationResult) { - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + const authResult = await authenticateEnvVarApiRequest(request, "read"); + if (!authResult.ok) { + return json({ error: authResult.error }, { status: authResult.status }); } + const authenticationResult = authResult.authentication; const environment = await authenticatedEnvironmentForAuthentication( authenticationResult, @@ -146,6 +144,10 @@ export async function loader({ params, request }: LoaderFunctionArgs) { const denied = await authorizeEnvVarApiRequest({ request, authType: authenticationResult.type, + ability: + authenticationResult.type === "apiKey" && authenticationResult.result.ok + ? authenticationResult.result.ability + : undefined, organizationId: environment.organizationId, projectId: environment.project.id, envType: environment.type, diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.import.ts b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.import.ts index 42b225f4e17..0f14d5b0f44 100644 --- a/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.import.ts +++ b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.import.ts @@ -4,11 +4,13 @@ import { ImportEnvironmentVariablesRequestBody } from "@trigger.dev/core/v3"; import { parse } from "dotenv"; import { z } from "zod"; import { - authenticateRequest, authenticatedEnvironmentForAuthentication, branchNameFromRequest, } from "~/services/apiAuth.server"; -import { authorizeEnvVarApiRequest } from "~/services/environmentVariableApiAccess.server"; +import { + authenticateEnvVarApiRequest, + authorizeEnvVarApiRequest, +} from "~/services/environmentVariableApiAccess.server"; import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; const ParamsSchema = z.object({ @@ -23,15 +25,11 @@ export async function action({ params, request }: ActionFunctionArgs) { return json({ error: "Invalid params" }, { status: 400 }); } - const authenticationResult = await authenticateRequest(request, { - personalAccessToken: true, - organizationAccessToken: true, - apiKey: true, - }); - - if (!authenticationResult) { - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + const authResult = await authenticateEnvVarApiRequest(request, "write"); + if (!authResult.ok) { + return json({ error: authResult.error }, { status: authResult.status }); } + const authenticationResult = authResult.authentication; const environment = await authenticatedEnvironmentForAuthentication( authenticationResult, @@ -43,6 +41,10 @@ export async function action({ params, request }: ActionFunctionArgs) { const denied = await authorizeEnvVarApiRequest({ request, authType: authenticationResult.type, + ability: + authenticationResult.type === "apiKey" && authenticationResult.result.ok + ? authenticationResult.result.ability + : undefined, organizationId: environment.organizationId, projectId: environment.project.id, envType: environment.type, diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.ts b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.ts index 49990204c1e..97b4795c25e 100644 --- a/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.ts +++ b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.ts @@ -3,11 +3,13 @@ import { json } from "@remix-run/server-runtime"; import { CreateEnvironmentVariableRequestBody } from "@trigger.dev/core/v3"; import { z } from "zod"; import { - authenticateRequest, authenticatedEnvironmentForAuthentication, branchNameFromRequest, } from "~/services/apiAuth.server"; -import { authorizeEnvVarApiRequest } from "~/services/environmentVariableApiAccess.server"; +import { + authenticateEnvVarApiRequest, + authorizeEnvVarApiRequest, +} from "~/services/environmentVariableApiAccess.server"; import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; const ParamsSchema = z.object({ @@ -22,15 +24,11 @@ export async function action({ params, request }: ActionFunctionArgs) { return json({ error: "Invalid params" }, { status: 400 }); } - const authenticationResult = await authenticateRequest(request, { - personalAccessToken: true, - organizationAccessToken: true, - apiKey: true, - }); - - if (!authenticationResult) { - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + const authResult = await authenticateEnvVarApiRequest(request, "write"); + if (!authResult.ok) { + return json({ error: authResult.error }, { status: authResult.status }); } + const authenticationResult = authResult.authentication; const environment = await authenticatedEnvironmentForAuthentication( authenticationResult, @@ -42,6 +40,10 @@ export async function action({ params, request }: ActionFunctionArgs) { const denied = await authorizeEnvVarApiRequest({ request, authType: authenticationResult.type, + ability: + authenticationResult.type === "apiKey" && authenticationResult.result.ok + ? authenticationResult.result.ability + : undefined, organizationId: environment.organizationId, projectId: environment.project.id, envType: environment.type, @@ -85,15 +87,11 @@ export async function loader({ params, request }: LoaderFunctionArgs) { return json({ error: "Invalid params" }, { status: 400 }); } - const authenticationResult = await authenticateRequest(request, { - personalAccessToken: true, - organizationAccessToken: true, - apiKey: true, - }); - - if (!authenticationResult) { - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + const authResult = await authenticateEnvVarApiRequest(request, "read"); + if (!authResult.ok) { + return json({ error: authResult.error }, { status: authResult.status }); } + const authenticationResult = authResult.authentication; const environment = await authenticatedEnvironmentForAuthentication( authenticationResult, @@ -105,6 +103,10 @@ export async function loader({ params, request }: LoaderFunctionArgs) { const denied = await authorizeEnvVarApiRequest({ request, authType: authenticationResult.type, + ability: + authenticationResult.type === "apiKey" && authenticationResult.result.ok + ? authenticationResult.result.ability + : undefined, organizationId: environment.organizationId, projectId: environment.project.id, envType: environment.type, diff --git a/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.ts b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.ts index 4fc8b680d1c..6e3f82f0df1 100644 --- a/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.ts +++ b/apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.ts @@ -2,7 +2,7 @@ import { type LoaderFunctionArgs, json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; import { env } from "~/env.server"; -import { authenticateApiRequest } from "~/services/apiAuth.server"; +import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; import { resolveVariablesForEnvironment } from "~/v3/environmentVariables/environmentVariablesRepository.server"; const ParamsSchema = z.object({ @@ -16,11 +16,16 @@ export async function loader({ request, params }: LoaderFunctionArgs) { return json({ error: "Invalid params" }, { status: 400 }); } - // Next authenticate the request - const authenticationResult = await authenticateApiRequest(request); - if (!authenticationResult) { - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + // Reading env vars requires the read:envvars scope (root keys authorize + // everything). + const authResult = await authenticateApiKeyWithScope(request, { + action: "read", + resource: { type: "envvars" }, + }); + if (!authResult.ok) { + return json({ error: authResult.error }, { status: authResult.status }); } + const authenticationResult = authResult.authentication; const { projectRef } = parsedParams.data; diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts index 8980aaeeb47..90f5772c5d3 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts @@ -28,6 +28,10 @@ const route = createActionApiRoute( params: z.object({ queueParam: z.string().transform((val) => val.replace(/%2F/g, "/")), }), + authorization: { + action: "write", + resource: () => ({ type: "queues" }), + }, }, async ({ params, body, authentication }) => { const input: RetrieveQueueParam = diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts index 9a83f3ed845..503d875e471 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts @@ -15,6 +15,10 @@ const route = createActionApiRoute( params: z.object({ queueParam: z.string().transform((val) => val.replace(/%2F/g, "/")), }), + authorization: { + action: "write", + resource: () => ({ type: "queues" }), + }, }, async ({ params, body, authentication }) => { const input: RetrieveQueueParam = diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.pause.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.pause.ts index a0f910fe58f..dd5f43da5d1 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.pause.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.pause.ts @@ -15,6 +15,10 @@ const route = createActionApiRoute( params: z.object({ queueParam: z.string().transform((val) => val.replace(/%2F/g, "/")), }), + authorization: { + action: "write", + resource: () => ({ type: "queues" }), + }, }, async ({ params, body, authentication }) => { const input: RetrieveQueueParam = diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.ts index a9bcd2342e5..44d54843d9d 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.ts @@ -15,6 +15,10 @@ export const loader = createLoaderApiRoute( }), searchParams: SearchParamsSchema, findResource: async () => 1, // This is a dummy function, we don't need to find a resource + authorization: { + action: "read", + resource: () => ({ type: "queues" }), + }, }, async ({ params, searchParams, authentication }) => { const input: RetrieveQueueParam = diff --git a/apps/webapp/app/routes/api.v1.queues.ts b/apps/webapp/app/routes/api.v1.queues.ts index b257a248a0b..27fe2eb93db 100644 --- a/apps/webapp/app/routes/api.v1.queues.ts +++ b/apps/webapp/app/routes/api.v1.queues.ts @@ -24,6 +24,7 @@ export const loader = createLoaderApiRoute( { searchParams: SearchParamsSchema, findResource: async () => 1, // This is a dummy function, we don't need to find a resource + authorization: { action: "read", resource: () => ({ type: "queues" }) }, }, async ({ searchParams, authentication }) => { const service = new QueueListPresenter(searchParams.perPage); diff --git a/apps/webapp/app/routes/api.v1.runs.$runParam.replay.ts b/apps/webapp/app/routes/api.v1.runs.$runParam.replay.ts index df684946b18..f0bf949e351 100644 --- a/apps/webapp/app/routes/api.v1.runs.$runParam.replay.ts +++ b/apps/webapp/app/routes/api.v1.runs.$runParam.replay.ts @@ -1,13 +1,13 @@ -import type { ActionFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import type { TaskRun } from "@trigger.dev/database"; import { z } from "zod"; import { prisma } from "~/db.server"; import { runStore } from "~/v3/runStore.server"; -import { authenticateApiRequest } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; +import { anyResource, createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; import { ReplayTaskRunService } from "~/v3/services/replayTaskRun.server"; import { findRunByIdWithMollifierFallback } from "~/v3/mollifier/readFallback.server"; +import { resolveRunForMutation } from "~/v3/mollifier/resolveRunForMutation.server"; import { sanitizeTriggerSource } from "~/utils/triggerSource"; import { clientSafeErrorMessage } from "~/utils/prismaErrors"; @@ -49,108 +49,116 @@ const BufferedReplayInputSchema = z.object({ seedMetadataType: z.string().nullable().optional(), }); -export async function action({ request, params }: ActionFunctionArgs) { - // Ensure this is a POST request - if (request.method.toUpperCase() !== "POST") { - return { status: 405, body: "Method Not Allowed" }; - } - - // Authenticate the request - const authenticationResult = await authenticateApiRequest(request); - if (!authenticationResult) { - return json({ error: "Invalid or Missing API Key" }, { status: 401 }); - } - - const parsed = ParamsSchema.safeParse(params); - if (!parsed.success) { - return json({ error: "Invalid or missing run ID" }, { status: 400 }); - } - - const { runParam } = parsed.data; +const { action } = createActionApiRoute( + { + params: ParamsSchema, + method: "POST", + findResource: (params, authentication) => + resolveRunForMutation({ + runParam: params.runParam, + environmentId: authentication.environment.id, + organizationId: authentication.environment.organizationId, + }), + authorization: { + action: "write", + resource: (params, _, __, ___, run) => + run + ? anyResource([ + { type: "runs", id: run.friendlyId }, + { type: "tasks", id: run.taskIdentifier }, + ]) + : { type: "runs", id: params.runParam }, + }, + }, + async ({ request, params, authentication }) => { + const { runParam } = params; - try { - const env = authenticationResult.environment; - // PG-first. Replay works on any status per audit — no - // filter beyond friendlyId is the existing semantic; findFirst with - // env scoping tightens it minimally without changing behaviour for - // a correctly-authed caller. - let taskRun: TaskRun | null = await runStore.findRun( - { - friendlyId: runParam, - runtimeEnvironmentId: env.id, - }, - prisma - ); + try { + const env = authentication.environment; + // PG-first. Replay works on any status per audit — no + // filter beyond friendlyId is the existing semantic; findFirst with + // env scoping tightens it minimally without changing behaviour for + // a correctly-authed caller. + let taskRun: TaskRun | null = await runStore.findRun( + { + friendlyId: runParam, + runtimeEnvironmentId: env.id, + }, + prisma + ); - if (!taskRun) { - // Buffered fallback. SyntheticRun carries every field - // ReplayTaskRunService reads from a TaskRun. Validate the subset of - // fields the service consumes (BufferedReplayInputSchema above) - // before casting; a schema mismatch surfaces as a 404 here rather - // than as a silent undefined deep inside the service. - const buffered = await findRunByIdWithMollifierFallback({ - runId: runParam, - environmentId: env.id, - organizationId: env.organizationId, - }); - if (buffered) { - const parsed = BufferedReplayInputSchema.safeParse(buffered); - if (parsed.success) { - // Manual sync point: `BufferedReplayInputSchema` covers only - // the subset of `TaskRun` fields `ReplayTaskRunService.call` - // currently reads from `existingTaskRun`. The cast is `as - // unknown as TaskRun` because the full `TaskRun` type carries - // ~40 fields the service never touches; mirroring all of them - // on a synthetic snapshot would be misleading. If a future - // change to `ReplayTaskRunService` reads an additional - // `existingTaskRun` field, **add it to the schema above** — - // otherwise the buffered path will silently feed the service - // `undefined` for that field while the PG-source replay - // works. The `safeParse` + warn-log + 404 below is the - // run-time fail-safe; this comment is the design fail-safe. - taskRun = parsed.data as unknown as TaskRun; - } else { - logger.warn("replay: buffered fallback failed schema validation", { - runParam, - issues: parsed.error.issues.map((issue) => ({ - path: issue.path.join("."), - code: issue.code, - })), - }); + if (!taskRun) { + // Buffered fallback. SyntheticRun carries every field + // ReplayTaskRunService reads from a TaskRun. Validate the subset of + // fields the service consumes (BufferedReplayInputSchema above) + // before casting; a schema mismatch surfaces as a 404 here rather + // than as a silent undefined deep inside the service. + const buffered = await findRunByIdWithMollifierFallback({ + runId: runParam, + environmentId: env.id, + organizationId: env.organizationId, + }); + if (buffered) { + const parsed = BufferedReplayInputSchema.safeParse(buffered); + if (parsed.success) { + // Manual sync point: `BufferedReplayInputSchema` covers only + // the subset of `TaskRun` fields `ReplayTaskRunService.call` + // currently reads from `existingTaskRun`. The cast is `as + // unknown as TaskRun` because the full `TaskRun` type carries + // ~40 fields the service never touches; mirroring all of them + // on a synthetic snapshot would be misleading. If a future + // change to `ReplayTaskRunService` reads an additional + // `existingTaskRun` field, **add it to the schema above** — + // otherwise the buffered path will silently feed the service + // `undefined` for that field while the PG-source replay + // works. The `safeParse` + warn-log + 404 below is the + // run-time fail-safe; this comment is the design fail-safe. + taskRun = parsed.data as unknown as TaskRun; + } else { + logger.warn("replay: buffered fallback failed schema validation", { + runParam, + issues: parsed.error.issues.map((issue) => ({ + path: issue.path.join("."), + code: issue.code, + })), + }); + } } } - } - if (!taskRun) { - return json({ error: "Run not found" }, { status: 404 }); - } + if (!taskRun) { + return json({ error: "Run not found" }, { status: 404 }); + } - const triggerSource = sanitizeTriggerSource(request.headers.get("x-trigger-source")) ?? "api"; + const triggerSource = sanitizeTriggerSource(request.headers.get("x-trigger-source")) ?? "api"; - const service = new ReplayTaskRunService(); - const newRun = await service.call(taskRun, { triggerSource }); + const service = new ReplayTaskRunService(); + const newRun = await service.call(taskRun, { triggerSource }); - if (!newRun) { - return json({ error: "Failed to create new run" }, { status: 400 }); - } + if (!newRun) { + return json({ error: "Failed to create new run" }, { status: 400 }); + } - return json({ - id: newRun?.friendlyId, - }); - } catch (error) { - if (error instanceof Error) { - logger.error("Failed to replay run", { - error: { - name: error.name, - message: error.message, - stack: error.stack, - }, - run: runParam, + return json({ + id: newRun?.friendlyId, }); - return json({ error: clientSafeErrorMessage(error) }, { status: 400 }); - } else { - logger.error("Failed to replay run", { error: JSON.stringify(error), run: runParam }); - return json({ error: JSON.stringify(error) }, { status: 400 }); + } catch (error) { + if (error instanceof Error) { + logger.error("Failed to replay run", { + error: { + name: error.name, + message: error.message, + stack: error.stack, + }, + run: runParam, + }); + return json({ error: clientSafeErrorMessage(error) }, { status: 400 }); + } else { + logger.error("Failed to replay run", { error: JSON.stringify(error), run: runParam }); + return json({ error: JSON.stringify(error) }, { status: 400 }); + } } } -} +); + +export { action }; diff --git a/apps/webapp/app/routes/api.v1.runs.ts b/apps/webapp/app/routes/api.v1.runs.ts index a4f543e68f5..dca246a0c24 100644 --- a/apps/webapp/app/routes/api.v1.runs.ts +++ b/apps/webapp/app/routes/api.v1.runs.ts @@ -3,7 +3,11 @@ import { ApiRunListPresenter, ApiRunListSearchParams, } from "~/presenters/v3/ApiRunListPresenter.server"; -import { anyResource, createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { + anyResource, + createLoaderApiRoute, + everyResource, +} from "~/services/routeBuilders/apiBuilder.server"; export const loader = createLoaderApiRoute( { @@ -18,17 +22,18 @@ export const loader = createLoaderApiRoute( // and the legacy `checkAuthorization` iterated `Object.keys` — so a // JWT with type-level `read:tasks` (no id) granted access to the // unfiltered runs list. The new ability model only matches against - // resources we list, so the type-level `{ type: "tasks" }` element - // (alongside `{ type: "runs" }` and the per-id task elements) - // preserves that semantic — `read:tasks` JWTs in the wild still - // list unfiltered runs without needing a separate `read:runs` - // scope. Per-id `read:tasks:foo` still grants only when the - // filter includes `foo`. - return anyResource([ - { type: "runs" }, - { type: "tasks" }, - ...taskFilter.map((id) => ({ type: "tasks", id })), - ]); + // resources we list. Keep type-level runs/tasks as alternatives so + // broad scopes retain that behavior. ID-scoped keys, however, must + // match every task in a multi-task filter; matching one item must not + // expose the others. + if (taskFilter.length === 0) { + return anyResource([{ type: "runs" }, { type: "tasks" }]); + } + + return everyResource( + taskFilter.map((id) => ({ type: "tasks", id })), + [{ type: "runs" }, { type: "tasks" }] + ); }, }, findResource: async () => 1, // This is a dummy function, we don't need to find a resource diff --git a/apps/webapp/app/routes/api.v1.sessions.ts b/apps/webapp/app/routes/api.v1.sessions.ts index 1a10e6c6f02..c18412c4810 100644 --- a/apps/webapp/app/routes/api.v1.sessions.ts +++ b/apps/webapp/app/routes/api.v1.sessions.ts @@ -27,6 +27,7 @@ import { anyResource, createActionApiRoute, createLoaderApiRoute, + everyResource, } from "~/services/routeBuilders/apiBuilder.server"; import { ServiceValidationError } from "~/v3/services/common.server"; import { runStore } from "~/v3/runStore.server"; @@ -48,15 +49,19 @@ export const loader = createLoaderApiRoute( // - Type-level `read:sessions` (the old superScope) matches the // sessions element (collection-level — no id) // - `read:all` / `admin` bypass via the JWT ability's wildcard branches - // The taskIdentifier filter accepts a string or an array; expand to - // one resource per task id so any per-task-scoped JWT among them - // grants access (the array gets OR semantics). + // The taskIdentifier filter accepts a string or an array. Broad + // sessions/tasks scopes remain alternatives, while ID-scoped keys must + // match every requested task so one allowed filter cannot expose others. resource: (_, __, searchParams) => { const taskFilter = asArray(searchParams["filter[taskIdentifier]"]) ?? []; - return anyResource([ - ...taskFilter.map((id) => ({ type: "tasks" as const, id })), - { type: "sessions" as const }, - ]); + if (taskFilter.length === 0) { + return anyResource([{ type: "sessions" as const }, { type: "tasks" as const }]); + } + + return everyResource( + taskFilter.map((id) => ({ type: "tasks" as const, id })), + [{ type: "sessions" as const }, { type: "tasks" as const }] + ); }, }, findResource: async () => 1, diff --git a/apps/webapp/app/routes/api.v1.tasks.$taskId.batch.ts b/apps/webapp/app/routes/api.v1.tasks.$taskId.batch.ts index e57092bc66b..2bd7bc56508 100644 --- a/apps/webapp/app/routes/api.v1.tasks.$taskId.batch.ts +++ b/apps/webapp/app/routes/api.v1.tasks.$taskId.batch.ts @@ -1,138 +1,134 @@ -import type { ActionFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import type { BatchTriggerTaskV2RequestBody } from "@trigger.dev/core/v3"; import { BatchTriggerTaskRequestBody } from "@trigger.dev/core/v3"; import { z } from "zod"; -import { fromZodError } from "zod-validation-error"; import { MAX_BATCH_TRIGGER_ITEMS } from "~/consts"; +import { canWriteParentRun } from "~/utils/parentRunAuthorization.server"; import { clientSafeErrorMessage } from "~/utils/prismaErrors"; import { env } from "~/env.server"; -import { authenticateApiRequest } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; +import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; import { BatchTriggerV3Service } from "~/v3/services/batchTriggerV3.server"; import { HeadersSchema } from "./api.v1.tasks.$taskId.trigger"; import { determineRealtimeStreamsVersion } from "~/services/realtime/v1StreamsGlobal.server"; +import { publicAccessTokenResponseHeaders } from "~/services/publicAccessTokenResponse.server"; const ParamsSchema = z.object({ taskId: z.string(), }); -export async function action({ request, params }: ActionFunctionArgs) { - // Ensure this is a POST request - if (request.method.toUpperCase() !== "POST") { - return { status: 405, body: "Method Not Allowed" }; - } - - // Next authenticate the request - const authenticationResult = await authenticateApiRequest(request); - - if (!authenticationResult) { - return json({ error: "Invalid or Missing API key" }, { status: 401 }); - } - - const rawHeaders = Object.fromEntries(request.headers); - - const headers = HeadersSchema.safeParse(rawHeaders); - - if (!headers.success) { - return json({ error: "Invalid headers" }, { status: 400 }); - } - - const { - "idempotency-key": idempotencyKey, - "trigger-version": triggerVersion, - "x-trigger-span-parent-as-link": spanParentAsLink, - "x-trigger-worker": isFromWorker, - "x-trigger-realtime-streams-version": realtimeStreamsVersion, - traceparent, - tracestate, - } = headers.data; - - const { taskId } = ParamsSchema.parse(params); - - const contentLength = request.headers.get("content-length"); - - if (!contentLength || parseInt(contentLength) > env.TASK_PAYLOAD_MAXIMUM_SIZE) { - return json({ error: "Request body too large" }, { status: 413 }); - } +const { action } = createActionApiRoute( + { + params: ParamsSchema, + headers: HeadersSchema, + body: BatchTriggerTaskRequestBody, + method: "POST", + maxContentLength: env.TASK_PAYLOAD_MAXIMUM_SIZE, + authorization: { + action: "batchTrigger", + resource: (params) => ({ type: "tasks", id: params.taskId }), + }, + }, + async ({ params, headers, body, authentication, ability }) => { + const { taskId } = params; + const { + "idempotency-key": idempotencyKey, + "trigger-version": triggerVersion, + "x-trigger-span-parent-as-link": spanParentAsLink, + "x-trigger-worker": isFromWorker, + "x-trigger-realtime-streams-version": realtimeStreamsVersion, + traceparent, + tracestate, + } = headers; + + logger.debug("Triggering batch", { + taskId, + idempotencyKey, + triggerVersion, + body, + }); - // Now parse the request body - const anyBody = await request.json(); + if (!body.items.length) { + return json({ error: "No items to trigger" }, { status: 400 }); + } - const body = BatchTriggerTaskRequestBody.safeParse(anyBody); + // Check the there are fewer than 100 items. This has to stay above the + // parent-run authorization below, which costs one lookup per distinct + // parent — an oversized batch must be rejected before it can spend them. + if (body.items.length > MAX_BATCH_TRIGGER_ITEMS) { + return json( + { + error: `Too many items. Maximum allowed batch size is ${MAX_BATCH_TRIGGER_ITEMS}.`, + }, + { status: 400 } + ); + } - if (!body.success) { - return json( - { error: fromZodError(body.error, { prefix: "Invalid batchTrigger call" }).toString() }, - { status: 400 } + const parentRunIds = Array.from( + new Set(body.items.map((item) => item.options?.parentRunId).filter((id) => id !== undefined)) ); - } - - logger.debug("Triggering batch", { - taskId, - idempotencyKey, - triggerVersion, - body: body.data, - }); - - if (!body.data.items.length) { - return json({ error: "No items to trigger" }, { status: 400 }); - } - - // Check the there are fewer than 100 items - if (body.data.items.length > MAX_BATCH_TRIGGER_ITEMS) { - return json( - { - error: `Too many items. Maximum allowed batch size is ${MAX_BATCH_TRIGGER_ITEMS}.`, - }, - { status: 400 } + const canWriteParentRuns = await Promise.all( + parentRunIds.map((parentRunId) => + canWriteParentRun( + ability, + authentication.environment.id, + authentication.environment.organizationId, + parentRunId + ) + ) ); - } + if (canWriteParentRuns.some((allowed) => !allowed)) { + return json({ error: "Unauthorized" }, { status: 403 }); + } - const service = new BatchTriggerV3Service(); + const service = new BatchTriggerV3Service(); - const traceContext = - traceparent && isFromWorker // If the request is from a worker, we should pass the trace context - ? { traceparent, tracestate } - : undefined; + const traceContext = + traceparent && isFromWorker // If the request is from a worker, we should pass the trace context + ? { traceparent, tracestate } + : undefined; - const v3Body = convertV1BodyToV2Body(body.data, taskId); + const v3Body = convertV1BodyToV2Body(body, taskId); - try { - const result = await service.call(authenticationResult.environment, v3Body, { - idempotencyKey: idempotencyKey ?? undefined, - triggerVersion: triggerVersion ?? undefined, - traceContext, - spanParentAsLink: spanParentAsLink === 1, - realtimeStreamsVersion: determineRealtimeStreamsVersion(realtimeStreamsVersion ?? undefined), - }); + try { + const result = await service.call(authentication.environment, v3Body, { + idempotencyKey: idempotencyKey ?? undefined, + triggerVersion: triggerVersion ?? undefined, + traceContext, + spanParentAsLink: spanParentAsLink === 1, + realtimeStreamsVersion: determineRealtimeStreamsVersion( + realtimeStreamsVersion ?? undefined + ), + }); - if (!result) { - return json({ error: "Task not found" }, { status: 404 }); - } + if (!result) { + return json({ error: "Task not found" }, { status: 404 }); + } - return json( - { - batchId: result.id, - runs: result.runs.map((run) => run.id), - }, - { - headers: { - "x-trigger-jwt-claims": JSON.stringify({ - sub: authenticationResult.environment.id, - pub: true, - }), + const $responseHeaders = await publicAccessTokenResponseHeaders({ + environment: authentication.environment, + scopes: [`read:batch:${result.id}`], + expirationTime: "1h", + }); + + return json( + { + batchId: result.id, + runs: result.runs.map((run) => run.id), }, + { headers: $responseHeaders } + ); + } catch (error) { + if (error instanceof Error) { + return json({ error: clientSafeErrorMessage(error) }, { status: 400 }); } - ); - } catch (error) { - if (error instanceof Error) { - return json({ error: clientSafeErrorMessage(error) }, { status: 400 }); - } - return json({ error: "Something went wrong" }, { status: 500 }); + return json({ error: "Something went wrong" }, { status: 500 }); + } } -} +); + +export { action }; // Strip from options: // - dependentBatch diff --git a/apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts b/apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts index 9d1e4c5b8dd..61650493303 100644 --- a/apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts +++ b/apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts @@ -20,6 +20,8 @@ import { handleRequestIdempotency, saveRequestIdempotency, } from "~/utils/requestIdempotency.server"; +import { scopeRequestIdempotencyKey } from "~/utils/requestIdempotencyKey"; +import { canWriteParentRun } from "~/utils/parentRunAuthorization.server"; import { sanitizeTriggerSource } from "~/utils/triggerSource"; import { runStore } from "~/v3/runStore.server"; import { ServiceValidationError } from "~/v3/services/baseService.server"; @@ -60,7 +62,7 @@ const { action, loader } = createActionApiRoute( }, corsStrategy: "all", }, - async ({ body, headers, params, authentication }) => { + async ({ body, headers, params, authentication, ability }) => { const { "idempotency-key": idempotencyKey, "idempotency-key-ttl": idempotencyKeyTTL, @@ -76,7 +78,22 @@ const { action, loader } = createActionApiRoute( "x-trigger-source": triggerSourceHeader, } = headers; - const cachedResponse = await handleRequestIdempotency(requestIdempotencyKey, { + if ( + !(await canWriteParentRun( + ability, + authentication.environment.id, + authentication.environment.organizationId, + body.options?.parentRunId + )) + ) { + return json({ error: "Unauthorized" }, { status: 403 }); + } + + const scopedIdempotencyKey = scopeRequestIdempotencyKey(requestIdempotencyKey, [ + authentication.environment.id, + params.taskId, + ]); + const cachedResponse = await handleRequestIdempotency(scopedIdempotencyKey, { requestType: "trigger", findCachedEntity: async (cachedRequestId) => { return await runStore.findRun( @@ -153,7 +170,7 @@ const { action, loader } = createActionApiRoute( // materialisation: once the run lands in PG, normal request- // idempotency from that point forward works as usual. if (!result.isMollified) { - await saveRequestIdempotency(requestIdempotencyKey, "trigger", result.run.id); + await saveRequestIdempotency(scopedIdempotencyKey, "trigger", result.run.id); } const $responseHeaders = await responseHeaders(result.run, authentication); diff --git a/apps/webapp/app/routes/api.v1.tasks.batch.ts b/apps/webapp/app/routes/api.v1.tasks.batch.ts index a43e0d3af53..5c9202d6fe5 100644 --- a/apps/webapp/app/routes/api.v1.tasks.batch.ts +++ b/apps/webapp/app/routes/api.v1.tasks.batch.ts @@ -1,8 +1,6 @@ import { json } from "@remix-run/server-runtime"; -import type { BatchTriggerTaskV2Response } from "@trigger.dev/core/v3"; -import { BatchTriggerTaskV2RequestBody, generateJWT } from "@trigger.dev/core/v3"; +import { BatchTriggerTaskV2RequestBody } from "@trigger.dev/core/v3"; import { env } from "~/env.server"; -import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { getOneTimeUseToken } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { createActionApiRoute, everyResource } from "~/services/routeBuilders/apiBuilder.server"; @@ -16,7 +14,7 @@ import { OutOfEntitlementError } from "~/v3/services/triggerTask.server"; import { sanitizeTriggerSource } from "~/utils/triggerSource"; import { HeadersSchema } from "./api.v1.tasks.$taskId.trigger"; import { determineRealtimeStreamsVersion } from "~/services/realtime/v1StreamsGlobal.server"; -import { extractJwtSigningSecretKey } from "~/services/realtime/jwtAuth.server"; +import { publicAccessTokenResponseHeaders } from "~/services/publicAccessTokenResponse.server"; const { action, loader } = createActionApiRoute( { @@ -124,11 +122,11 @@ const { action, loader } = createActionApiRoute( triggerAction: "trigger", }); - const $responseHeaders = await responseHeaders( - batch, - authentication.environment, - triggerClient - ); + const $responseHeaders = await publicAccessTokenResponseHeaders({ + environment: authentication.environment, + scopes: [`read:batch:${batch.id}`], + expirationTime: "1h", + }); return json(batch, { status: 202, headers: $responseHeaders }); } catch (error) { @@ -163,38 +161,4 @@ const { action, loader } = createActionApiRoute( } ); -async function responseHeaders( - batch: BatchTriggerTaskV2Response, - environment: AuthenticatedEnvironment, - triggerClient?: string | null -): Promise<Record<string, string>> { - const claimsHeader = JSON.stringify({ - sub: environment.id, - pub: true, - }); - - if (triggerClient === "browser") { - const claims = { - sub: environment.id, - pub: true, - scopes: [`read:batch:${batch.id}`], - }; - - const jwt = await generateJWT({ - secretKey: extractJwtSigningSecretKey(environment), - payload: claims, - expirationTime: "1h", - }); - - return { - "x-trigger-jwt-claims": claimsHeader, - "x-trigger-jwt": jwt, - }; - } - - return { - "x-trigger-jwt-claims": claimsHeader, - }; -} - export { action, loader }; diff --git a/apps/webapp/app/routes/api.v1.waitpoints.tokens.ts b/apps/webapp/app/routes/api.v1.waitpoints.tokens.ts index ace6128ac7e..62322c527c7 100644 --- a/apps/webapp/app/routes/api.v1.waitpoints.tokens.ts +++ b/apps/webapp/app/routes/api.v1.waitpoints.tokens.ts @@ -15,10 +15,10 @@ import { runOpsSplitReadEnabled, type PrismaClientOrTransaction, } from "~/db.server"; -import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { resolveRunIdMintKind } from "~/v3/engineVersion.server"; import { logger } from "~/services/logger.server"; import { generateHttpCallbackUrl } from "~/services/httpCallback.server"; +import { publicAccessTokenResponseHeaders } from "~/services/publicAccessTokenResponse.server"; import { createActionApiRoute, createLoaderApiRoute, @@ -103,11 +103,16 @@ const { action } = createActionApiRoute( standaloneResidency: residency, }); - const $responseHeaders = await responseHeaders(authentication.environment); + const waitpointId = WaitpointId.toFriendlyId(result.waitpoint.id); + const $responseHeaders = await publicAccessTokenResponseHeaders({ + environment: authentication.environment, + scopes: [`write:waitpoints:${waitpointId}`], + expirationTime: "24h", + }); return json<CreateWaitpointTokenResponseBody>( { - id: WaitpointId.toFriendlyId(result.waitpoint.id), + id: waitpointId, isCached: result.isCached, url: generateHttpCallbackUrl(result.waitpoint.id, authentication.environment.apiKey), }, @@ -124,17 +129,4 @@ const { action } = createActionApiRoute( } ); -async function responseHeaders( - environment: AuthenticatedEnvironment -): Promise<Record<string, string>> { - const claimsHeader = JSON.stringify({ - sub: environment.id, - pub: true, - }); - - return { - "x-trigger-jwt-claims": claimsHeader, - }; -} - export { action }; diff --git a/apps/webapp/app/routes/api.v2.deployments.$deploymentId.finalize.ts b/apps/webapp/app/routes/api.v2.deployments.$deploymentId.finalize.ts index 0fece044fb6..b7e6bdc2f27 100644 --- a/apps/webapp/app/routes/api.v2.deployments.$deploymentId.finalize.ts +++ b/apps/webapp/app/routes/api.v2.deployments.$deploymentId.finalize.ts @@ -2,7 +2,7 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { FinalizeDeploymentRequestBody } from "@trigger.dev/core/v3"; import { z } from "zod"; -import { authenticateApiRequest } from "~/services/apiAuth.server"; +import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { ServiceValidationError } from "~/v3/services/baseService.server"; import { FinalizeDeploymentV2Service } from "~/v3/services/finalizeDeploymentV2.server"; @@ -24,13 +24,18 @@ export async function action({ request, params }: ActionFunctionArgs) { } // Next authenticate the request - const authenticationResult = await authenticateApiRequest(request); + const authResult = await authenticateApiKeyWithScope(request, { + action: "write", + resource: { type: "deployments" }, + }); - if (!authenticationResult) { + if (!authResult.ok) { logger.info("Invalid or missing api key", { url: request.url }); - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + return json({ error: authResult.error }, { status: authResult.status }); } + const authenticationResult = authResult.authentication; + const authenticatedEnv = authenticationResult.environment; const { deploymentId } = parsedParams.data; diff --git a/apps/webapp/app/routes/api.v2.runs.$runParam.cancel.ts b/apps/webapp/app/routes/api.v2.runs.$runParam.cancel.ts index cb660049c9c..18a5d2cc30f 100644 --- a/apps/webapp/app/routes/api.v2.runs.$runParam.cancel.ts +++ b/apps/webapp/app/routes/api.v2.runs.$runParam.cancel.ts @@ -1,7 +1,7 @@ import { json } from "@remix-run/server-runtime"; import { z } from "zod"; import { env as appEnv } from "~/env.server"; -import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { anyResource, createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; import { getRequestAbortSignal } from "~/services/httpAsyncStorage.server"; import { CancelTaskRunService } from "~/v3/services/cancelTaskRun.server"; import { mutateWithFallback } from "~/v3/mollifier/mutateWithFallback.server"; @@ -19,10 +19,6 @@ const { action } = createActionApiRoute( params: ParamsSchema, allowJWT: true, corsStrategy: "none", - authorization: { - action: "write", - resource: (params) => ({ type: "runs", id: params.runParam }), - }, // PG-or-buffer resolver. Returning null here would 404 BEFORE the // action runs (`apiBuilder.server.ts:321`), so buffered cancels need // a buffer check at this layer too. Logic lives in a helper so the @@ -36,6 +32,16 @@ const { action } = createActionApiRoute( environmentId: auth.environment.id, organizationId: auth.environment.organizationId, }), + authorization: { + action: "write", + resource: (params, _, __, ___, run) => + run + ? anyResource([ + { type: "runs", id: run.friendlyId }, + { type: "tasks", id: run.taskIdentifier }, + ]) + : { type: "runs", id: params.runParam }, + }, }, async ({ params, authentication }) => { const runId = params.runParam; diff --git a/apps/webapp/app/routes/api.v2.tasks.batch.ts b/apps/webapp/app/routes/api.v2.tasks.batch.ts index c766bb474f6..5dcbf13e0fb 100644 --- a/apps/webapp/app/routes/api.v2.tasks.batch.ts +++ b/apps/webapp/app/routes/api.v2.tasks.batch.ts @@ -12,6 +12,8 @@ import { handleRequestIdempotency, saveRequestIdempotency, } from "~/utils/requestIdempotency.server"; +import { scopeRequestIdempotencyKey } from "~/utils/requestIdempotencyKey"; +import { canWriteParentRun } from "~/utils/parentRunAuthorization.server"; import { ServiceValidationError } from "~/v3/services/baseService.server"; import { BatchProcessingStrategy } from "~/v3/services/batchTriggerV3.server"; import { OutOfEntitlementError } from "~/v3/services/triggerTask.server"; @@ -45,11 +47,22 @@ const { action, loader } = createActionApiRoute( }, corsStrategy: "all", }, - async ({ body, headers, params, authentication }) => { + async ({ body, headers, params, authentication, ability }) => { if (!body.items.length) { return json({ error: "Batch cannot be triggered with no items" }, { status: 400 }); } + if ( + !(await canWriteParentRun( + ability, + authentication.environment.id, + authentication.environment.organizationId, + body.parentRunId + )) + ) { + return json({ error: "Unauthorized" }, { status: 403 }); + } + // Check the there are fewer than MAX_BATCH_V2_TRIGGER_ITEMS items if (body.items.length > env.MAX_BATCH_V2_TRIGGER_ITEMS) { return json( @@ -87,7 +100,11 @@ const { action, loader } = createActionApiRoute( requestIdempotencyKey, }); - const cachedResponse = await handleRequestIdempotency(requestIdempotencyKey, { + const scopedIdempotencyKey = scopeRequestIdempotencyKey(requestIdempotencyKey, [ + authentication.environment.id, + ...Array.from(new Set(body.items.map((item) => item.task))).sort(), + ]); + const cachedResponse = await handleRequestIdempotency(scopedIdempotencyKey, { requestType: "batch-trigger", findCachedEntity: async (cachedRequestId) => { const batch = await runStore.findBatchTaskRunById(cachedRequestId); @@ -99,7 +116,7 @@ const { action, loader } = createActionApiRoute( runCount: cachedBatch.runCount, }), buildResponseHeaders: async (responseBody, cachedEntity) => { - return await responseHeaders(responseBody, authentication.environment, triggerClient); + return await responseHeaders(responseBody, authentication.environment); }, }); @@ -116,7 +133,7 @@ const { action, loader } = createActionApiRoute( const service = new RunEngineBatchTriggerService(batchProcessingStrategy ?? undefined); service.onBatchTaskRunCreated.attachOnce(async (batch) => { - await saveRequestIdempotency(requestIdempotencyKey, "batch-trigger", batch.id); + await saveRequestIdempotency(scopedIdempotencyKey, "batch-trigger", batch.id); }); try { @@ -132,11 +149,7 @@ const { action, loader } = createActionApiRoute( triggerAction: "trigger", }); - const $responseHeaders = await responseHeaders( - batch, - authentication.environment, - triggerClient - ); + const $responseHeaders = await responseHeaders(batch, authentication.environment); return json(batch, { status: 202, @@ -176,35 +189,23 @@ const { action, loader } = createActionApiRoute( async function responseHeaders( batch: BatchTriggerTaskV3Response, - environment: AuthenticatedEnvironment, - triggerClient?: string | null + environment: AuthenticatedEnvironment ): Promise<Record<string, string>> { - const claimsHeader = JSON.stringify({ + const claims = { sub: environment.id, pub: true, - }); - - if (triggerClient === "browser") { - const claims = { - sub: environment.id, - pub: true, - scopes: [`read:batch:${batch.id}`], - }; - - const jwt = await generateJWT({ - secretKey: extractJwtSigningSecretKey(environment), - payload: claims, - expirationTime: "1h", - }); + scopes: [`read:batch:${batch.id}`], + }; - return { - "x-trigger-jwt-claims": claimsHeader, - "x-trigger-jwt": jwt, - }; - } + const jwt = await generateJWT({ + secretKey: extractJwtSigningSecretKey(environment), + payload: claims, + expirationTime: "1h", + }); return { - "x-trigger-jwt-claims": claimsHeader, + "x-trigger-jwt-claims": JSON.stringify({ sub: environment.id, pub: true }), + "x-trigger-jwt": jwt, }; } diff --git a/apps/webapp/app/routes/api.v3.batches.$batchId.items.ts b/apps/webapp/app/routes/api.v3.batches.$batchId.items.ts index c34dbc178c4..9e2bd510256 100644 --- a/apps/webapp/app/routes/api.v3.batches.$batchId.items.ts +++ b/apps/webapp/app/routes/api.v3.batches.$batchId.items.ts @@ -6,8 +6,12 @@ import { createNdjsonParserStream, streamToAsyncIterable, } from "~/runEngine/services/streamBatchItems.server"; -import { authenticateApiRequestWithFailure } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; +import { rbac } from "~/services/rbac.server"; +import { + authorizedBatchItemStream, + BatchItemAuthorizationError, +} from "~/utils/batchItemAuthorization"; import { ServiceValidationError } from "~/v3/services/baseService.server"; const ParamsSchema = z.object({ @@ -48,13 +52,22 @@ export async function action({ request, params }: ActionFunctionArgs) { ); } - // Authenticate the request - const authResult = await authenticateApiRequestWithFailure(request, { - allowPublicKey: true, - }); + // This streaming route cannot use createActionApiRoute because the body must + // remain an unread stream. Use the same RBAC controller directly and apply + // its ability to every parsed item below. + // + // Because we bypass the route builder, we also bypass its + // `restrictedApiKey && !authorization -> 403` fail-closed. Authorization is + // instead enforced per item by `authorizedBatchItemStream` below, which also + // requires a restricted credential to present at least one authorized item + // before the service may touch the batch — otherwise an empty stream would + // reach it having passed no checks at all. Any logic added between here and + // that call runs authenticated but NOT authorized, so keep new per-request + // work behind it. + const authResult = await rbac.authenticateBearer(request, { allowJWT: true }); if (!authResult.ok) { - return json({ error: authResult.error }, { status: 401 }); + return json({ error: authResult.error }, { status: authResult.status }); } // Get the request body stream @@ -76,8 +89,14 @@ export async function action({ request, params }: ActionFunctionArgs) { // Pipe the request body through the parser const parsedStream = body.pipeThrough(parser); - // Convert to async iterable for the service - const itemsIterator = streamToAsyncIterable(parsedStream); + // Convert to async iterable for the service. This authorizes the first item + // eagerly, so a stream that yields no items is rejected before the service + // can report anything about the batch. + const itemsIterator = await authorizedBatchItemStream( + streamToAsyncIterable(parsedStream), + authResult.ability, + batchId + ); // Process the stream const service = new StreamBatchItemsService(); @@ -88,6 +107,10 @@ export async function action({ request, params }: ActionFunctionArgs) { return json(result, { status: 200 }); } catch (error) { + if (error instanceof BatchItemAuthorizationError) { + return json({ error: "Unauthorized" }, { status: 403 }); + } + // Customer-facing validation failures (invalid item shape, invalid JSON // in the streamed body). The handler returns 4xx with the message; // system handles it gracefully, no alert needed. diff --git a/apps/webapp/app/routes/api.v3.batches.ts b/apps/webapp/app/routes/api.v3.batches.ts index 3179e2fa6aa..071bb783b8b 100644 --- a/apps/webapp/app/routes/api.v3.batches.ts +++ b/apps/webapp/app/routes/api.v3.batches.ts @@ -5,16 +5,24 @@ import { env } from "~/env.server"; import { BatchRateLimitExceededError } from "~/runEngine/concerns/batchLimits.server"; import { CreateBatchService } from "~/runEngine/services/createBatch.server"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import type { RbacAbility } from "@trigger.dev/rbac"; import { getOneTimeUseToken } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { extractJwtSigningSecretKey } from "~/services/realtime/jwtAuth.server"; import { determineRealtimeStreamsVersion } from "~/services/realtime/v1StreamsGlobal.server"; -import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { + anyResource, + createActionApiRoute, + everyResource, +} from "~/services/routeBuilders/apiBuilder.server"; +import { batchPublicAccessScopes } from "~/utils/batchItemAuthorization"; +import { canWriteParentRun } from "~/utils/parentRunAuthorization.server"; import { clientSafeErrorMessage } from "~/utils/prismaErrors"; import { handleRequestIdempotency, saveRequestIdempotency, } from "~/utils/requestIdempotency.server"; +import { scopeRequestIdempotencyKey } from "~/utils/requestIdempotencyKey"; import { sanitizeTriggerSource } from "~/utils/triggerSource"; import { ServiceValidationError } from "~/v3/services/baseService.server"; import { OutOfEntitlementError } from "~/v3/services/triggerTask.server"; @@ -37,18 +45,41 @@ const { action, loader } = createActionApiRoute( maxContentLength: 131_072, // 128KB is plenty for the batch metadata authorization: { action: "batchTrigger", - // No specific tasks to authorize at batch creation time — tasks are - // validated when items are streamed. Collection-level check. - resource: () => ({ type: "tasks" }), + resource: (_params, _searchParams, _headers, body) => { + // Newer clients declare the distinct task identifiers before creating + // the batch, so selected-task credentials can be authorized before the + // shell is created. Older clients omit them and retain the existing + // collection-level behavior: broad credentials pass, selected-task + // credentials fail closed. + if (!body.taskIdentifiers) { + return anyResource([{ type: "batch" }, { type: "tasks" }]); + } + + return everyResource( + body.taskIdentifiers.map((id) => ({ type: "tasks" as const, id })), + [{ type: "batch" }, { type: "tasks" }] + ); + }, }, corsStrategy: "all", }, - async ({ body, headers, authentication }) => { + async ({ body, headers, authentication, ability }) => { // Validate runCount if (body.runCount <= 0) { return json({ error: "runCount must be a positive integer" }, { status: 400 }); } + if ( + !(await canWriteParentRun( + ability, + authentication.environment.id, + authentication.environment.organizationId, + body.parentRunId + )) + ) { + return json({ error: "Unauthorized" }, { status: 403 }); + } + // Check runCount against limit if (body.runCount > env.STREAMING_BATCH_MAX_ITEMS) { return json( @@ -82,11 +113,18 @@ const { action, loader } = createActionApiRoute( triggerClient, }); - // Handle idempotency for the batch creation + // Keep create-batch retries isolated by environment and the task set that + // was authorized above. Sorting makes the scope stable when callers send + // the same identifiers in a different order. + const scopedIdempotencyKey = scopeRequestIdempotencyKey(body.idempotencyKey, [ + authentication.environment.id, + ...(body.taskIdentifiers ? [...new Set(body.taskIdentifiers)].sort() : []), + ]); + const cachedResponse = await handleRequestIdempotency< { friendlyId: string; runCount: number }, CreateBatchResponse - >(body.idempotencyKey, { + >(scopedIdempotencyKey, { requestType: "create-batch", findCachedEntity: async (cachedRequestId) => { const batch = await engine.runStore.findBatchTaskRunById(cachedRequestId); @@ -99,7 +137,12 @@ const { action, loader } = createActionApiRoute( isCached: true, }), buildResponseHeaders: async (responseBody) => { - return await responseHeaders(responseBody, authentication.environment, triggerClient); + return await responseHeaders( + responseBody, + authentication.environment, + ability, + triggerClient + ); }, }); @@ -114,7 +157,7 @@ const { action, loader } = createActionApiRoute( const service = new CreateBatchService(); service.onBatchTaskRunCreated.attachOnce(async (batch) => { - await saveRequestIdempotency(body.idempotencyKey, "create-batch", batch.id); + await saveRequestIdempotency(scopedIdempotencyKey, "create-batch", batch.id); }); try { @@ -132,6 +175,7 @@ const { action, loader } = createActionApiRoute( const $responseHeaders = await responseHeaders( batch, authentication.environment, + ability, triggerClient ); @@ -198,34 +242,27 @@ const { action, loader } = createActionApiRoute( async function responseHeaders( batch: CreateBatchResponse, environment: AuthenticatedEnvironment, + ability: RbacAbility, triggerClient?: string | null ): Promise<Record<string, string>> { - const claimsHeader = JSON.stringify({ - sub: environment.id, - pub: true, - }); + // Browser clients need a delegated token for phase two because they must not + // retain a private API key. Selected-task credentials only receive read access + // and must continue to authorize each streamed item with their private key. + const scopes = batchPublicAccessScopes(batch.id, ability, triggerClient === "browser"); - if (triggerClient === "browser") { - const claims = { + const jwt = await generateJWT({ + secretKey: extractJwtSigningSecretKey(environment), + payload: { sub: environment.id, pub: true, - scopes: [`read:batch:${batch.id}`, `write:batch:${batch.id}`], - }; - - const jwt = await generateJWT({ - secretKey: extractJwtSigningSecretKey(environment), - payload: claims, - expirationTime: "1h", - }); - - return { - "x-trigger-jwt-claims": claimsHeader, - "x-trigger-jwt": jwt, - }; - } + scopes, + }, + expirationTime: "1h", + }); return { - "x-trigger-jwt-claims": claimsHeader, + "x-trigger-jwt-claims": JSON.stringify({ sub: environment.id, pub: true }), + "x-trigger-jwt": jwt, }; } diff --git a/apps/webapp/app/routes/api.v3.deployments.$deploymentId.finalize.ts b/apps/webapp/app/routes/api.v3.deployments.$deploymentId.finalize.ts index 87fff8bbf76..2fdcc0de21e 100644 --- a/apps/webapp/app/routes/api.v3.deployments.$deploymentId.finalize.ts +++ b/apps/webapp/app/routes/api.v3.deployments.$deploymentId.finalize.ts @@ -2,7 +2,7 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { FinalizeDeploymentRequestBody } from "@trigger.dev/core/v3"; import { z } from "zod"; -import { authenticateApiRequest } from "~/services/apiAuth.server"; +import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { ServiceValidationError } from "~/v3/services/baseService.server"; import { FinalizeDeploymentV2Service } from "~/v3/services/finalizeDeploymentV2.server"; @@ -24,13 +24,18 @@ export async function action({ request, params }: ActionFunctionArgs) { } // Next authenticate the request - const authenticationResult = await authenticateApiRequest(request); + const authResult = await authenticateApiKeyWithScope(request, { + action: "write", + resource: { type: "deployments" }, + }); - if (!authenticationResult) { + if (!authResult.ok) { logger.info("Invalid or missing api key", { url: request.url }); - return json({ error: "Invalid or Missing API key" }, { status: 401 }); + return json({ error: authResult.error }, { status: authResult.status }); } + const authenticationResult = authResult.authentication; + const authenticatedEnv = authenticationResult.environment; const { deploymentId } = parsedParams.data; diff --git a/apps/webapp/app/services/apiAuth.server.ts b/apps/webapp/app/services/apiAuth.server.ts index 899b693a98e..90fc9d2d3b7 100644 --- a/apps/webapp/app/services/apiAuth.server.ts +++ b/apps/webapp/app/services/apiAuth.server.ts @@ -9,9 +9,11 @@ import { authIncludeBase, authIncludeWithParent, findEnvironmentByApiKey, + findEnvironmentByApiKeyWithResolution, findEnvironmentByPublicApiKey, toAuthenticated, } from "~/models/runtimeEnvironment.server"; +import type { RbacAbility, RbacResource } from "@trigger.dev/rbac"; import { type RuntimeEnvironmentForEnvRepo } from "~/v3/environmentVariables/environmentVariablesRepository.server"; import { logger } from "./logger.server"; import { safeEnvironmentLogFields } from "./safeEnvironmentLog"; @@ -28,6 +30,10 @@ import { } from "./organizationAccessToken.server"; import { isPublicJWT, validatePublicJwtKey } from "./realtime/jwtAuth.server"; import { isDefaultDevBranch, sanitizeBranchName } from "@trigger.dev/core/v3/utils/gitBranch"; +import { + authenticateAuthorizeBearerWithTelemetry, + observeLegacyBearerAuthentication, +} from "~/services/authTelemetry.server"; const ClaimsSchema = z.object({ scopes: z.array(z.string()).optional(), @@ -58,6 +64,10 @@ export type ApiAuthenticationResultSuccess = { realtime?: { skipColumns?: string[]; }; + // Present when authentication went through the RBAC bearer controller. + // Legacy direct authentication intentionally omits it and remains fail-closed + // for restricted additional keys. + ability?: RbacAbility; // Present when the request used a public JWT minted from a PAT/UAT exchange // that stamped an `act` delegation claim. `actor.sub` is the acting user id, // used for attribution (e.g. who resolved an error). Absent for plain env @@ -85,9 +95,9 @@ export async function authenticateApiRequest( return; } - const authentication = await authenticateApiKey(apiKey, { ...options, branchName }); - - return authentication; + return observeLegacyBearerAuthentication(request, () => + authenticateApiKey(apiKey, { ...options, branchName }) + ); } /** @@ -107,9 +117,9 @@ export async function authenticateApiRequestWithFailure( }; } - const authentication = await authenticateApiKeyWithFailure(apiKey, { ...options, branchName }); - - return authentication; + return observeLegacyBearerAuthentication(request, () => + authenticateApiKeyWithFailure(apiKey, { ...options, branchName }) + ); } /** @@ -117,7 +127,11 @@ export async function authenticateApiRequestWithFailure( */ export async function authenticateApiKey( apiKey: string, - options: { allowPublicKey?: boolean; allowJWT?: boolean; branchName?: string } = {} + options: { + allowPublicKey?: boolean; + allowJWT?: boolean; + branchName?: string; + } = {} ): Promise<ApiAuthenticationResultSuccess | undefined> { const result = getApiKeyResult(apiKey); @@ -184,7 +198,11 @@ export async function authenticateApiKey( */ async function authenticateApiKeyWithFailure( apiKey: string, - options: { allowPublicKey?: boolean; allowJWT?: boolean; branchName?: string } = {} + options: { + allowPublicKey?: boolean; + allowJWT?: boolean; + branchName?: string; + } = {} ): Promise<ApiAuthenticationResult> { const result = getApiKeyResult(apiKey); @@ -226,18 +244,24 @@ async function authenticateApiKeyWithFailure( }; } case "PRIVATE": { - const environment = await findEnvironmentByApiKey(result.apiKey, options.branchName); - if (!environment) { + const resolution = await findEnvironmentByApiKeyWithResolution( + result.apiKey, + options.branchName + ); + if (!resolution.ok) { return { ok: false, - error: "Invalid API Key", + error: + resolution.reason === "restricted" + ? "This endpoint does not support restricted API keys. Use an API key with full environment access." + : "Invalid API Key", }; } return { ok: true, ...result, - environment, + environment: resolution.environment, }; } case "PUBLIC_JWT": { @@ -260,6 +284,52 @@ async function authenticateApiKeyWithFailure( } } +/** + * Authenticate an API-key request for a legacy (non-apiBuilder) route that + * needs to accept granular additional keys, then enforce that the key's ability + * authorizes `action` on `resource`. Root keys (and grace-window root keys) + * carry the unrestricted `admin` ability, preserving pre-granular behavior. + * + * Only apiKey credentials are accepted (no PAT / org token / public key). Use + * this for routes previously guarded by a bare `authenticateApiRequest` call. + */ +export async function authenticateApiKeyWithScope( + request: Request, + { + action, + resource, + allowJWT = false, + }: { action: string; resource: RbacResource; allowJWT?: boolean } +): Promise< + | { ok: true; authentication: ApiAuthenticationResultSuccess } + | { ok: false; status: 401 | 403; error: string } +> { + const apiKey = getApiKeyFromHeader(request.headers.get("Authorization")); + if (!apiKey) { + return { ok: false, status: 401, error: "Invalid or Missing API key" }; + } + + const result = await authenticateAuthorizeBearerWithTelemetry( + request, + { action, resource }, + { allowJWT } + ); + if (!result.ok) { + return result; + } + + return { + ok: true, + authentication: { + ok: true, + apiKey, + type: "PRIVATE", + environment: result.environment, + ability: result.ability, + }, + }; +} + export async function authenticateAuthorizationHeader( authorization: string, { @@ -434,7 +504,10 @@ export async function authenticateRequest< } if (allowedMethods.apiKey) { - const result = await authenticateApiKey(apiKey, { allowPublicKey: false, branchName }); + const result = await authenticateApiKey(apiKey, { + allowPublicKey: false, + branchName, + }); if (!result) { return; diff --git a/apps/webapp/app/services/authFeatureControls.server.ts b/apps/webapp/app/services/authFeatureControls.server.ts new file mode 100644 index 00000000000..5052b6b5a3c --- /dev/null +++ b/apps/webapp/app/services/authFeatureControls.server.ts @@ -0,0 +1,13 @@ +import { resolveAuthFeatureControls } from "~/services/authFeatureControls"; +import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server"; + +export { resolveAuthFeatureControls } from "~/services/authFeatureControls"; +export type { AuthFeatureControls } from "~/services/authFeatureControls"; + +function currentControls() { + return resolveAuthFeatureControls(globalFlagsRegistry.current()); +} + +export const authFeatureControls = { + additionalApiKeyLookupEnabled: () => currentControls().additionalApiKeyLookupEnabled, +}; diff --git a/apps/webapp/app/services/authFeatureControls.ts b/apps/webapp/app/services/authFeatureControls.ts new file mode 100644 index 00000000000..1707dee37fb --- /dev/null +++ b/apps/webapp/app/services/authFeatureControls.ts @@ -0,0 +1,13 @@ +import { FEATURE_FLAG, type FeatureFlagCatalog } from "~/v3/featureFlags"; + +export type AuthFeatureControls = { + additionalApiKeyLookupEnabled: boolean; +}; + +export function resolveAuthFeatureControls( + flags: Partial<FeatureFlagCatalog> | Record<string, unknown> | undefined +): AuthFeatureControls { + return { + additionalApiKeyLookupEnabled: flags?.[FEATURE_FLAG.additionalApiKeyLookupEnabled] === true, + }; +} diff --git a/apps/webapp/app/services/authTelemetry.server.ts b/apps/webapp/app/services/authTelemetry.server.ts new file mode 100644 index 00000000000..378fe4e7606 --- /dev/null +++ b/apps/webapp/app/services/authTelemetry.server.ts @@ -0,0 +1,168 @@ +import { getMeter } from "@internal/tracing"; +import { isAdditionalApiKey } from "@trigger.dev/core/v3/apiKeys"; +import { isPublicJWT } from "@trigger.dev/core/v3/jwt"; +import type { + BearerCredentialKind, + BearerLookupPath, + HostBearerAuthResult, + RbacResource, +} from "@trigger.dev/rbac"; +import { authFeatureControls } from "~/services/authFeatureControls.server"; +import { rbac } from "~/services/rbac.server"; +import { singleton } from "~/utils/singleton"; + +export type ApiAuthResult = "success" | "invalid" | "forbidden" | "disabled" | "error"; + +const telemetry = singleton("apiAuthTelemetry", () => { + const meter = getMeter("api-auth"); + const attempts = meter.createCounter("api_auth.attempts", { + description: "Completed environment bearer authentication attempts", + }); + const duration = meter.createHistogram("api_auth.duration_ms", { + description: "Environment bearer authentication duration", + unit: "ms", + }); + + meter + .createObservableGauge("api_auth.rollout_mode", { + description: "Active API authentication rollout modes", + }) + .addCallback((result) => { + result.observe(1, { + control: "additional_key_lookup", + mode: authFeatureControls.additionalApiKeyLookupEnabled() ? "enabled" : "disabled", + }); + }); + + return { attempts, duration }; +}); + +export async function authenticateBearerWithTelemetry( + request: Request, + options: { allowJWT: boolean } +): Promise<HostBearerAuthResult> { + const startedAt = performance.now(); + const classified = classifyCredential(request, options.allowJWT); + let final = { ...classified, result: "error" as ApiAuthResult }; + + try { + const result = await rbac.authenticateBearer(request, options); + // The host LazyController always attaches `resolution`; fall back to the + // format-based classification if a caller (e.g. a test double) omits it. + const resolution = result.resolution ?? classified; + final = { + credentialKind: resolution.credentialKind, + lookupPath: resolution.lookupPath, + result: result.ok + ? "success" + : resolution.lookupPath === "additional_skipped" + ? "disabled" + : result.status === 403 + ? "forbidden" + : "invalid", + }; + recordAuthAttempt("rbac", final.credentialKind, final.lookupPath, final.result); + return result; + } catch (error) { + recordAuthAttempt("rbac", final.credentialKind, final.lookupPath, final.result); + throw error; + } finally { + telemetry.duration.record(performance.now() - startedAt, { + resolver: "rbac", + credential_kind: final.credentialKind, + result: final.result, + lookup_path: final.lookupPath, + }); + } +} + +export async function authenticateAuthorizeBearerWithTelemetry( + request: Request, + check: { action: string; resource: RbacResource }, + options: { allowJWT: boolean } +) { + // Keep authentication telemetry consistent with apiBuilder: a valid + // credential records a successful authentication even when the subsequent + // resource authorization fails. Authorization correctness is covered by the + // route tests rather than folded into the authentication-health metric. + const result = await authenticateBearerWithTelemetry(request, options); + if (!result.ok) return result; + + if (!result.ability.can(check.action, check.resource)) { + return { ok: false as const, status: 403 as const, error: "Unauthorized" }; + } + + return result; +} + +export async function observeLegacyBearerAuthentication<T extends { ok: boolean } | undefined>( + request: Request, + operation: () => Promise<T> +): Promise<T> { + const startedAt = performance.now(); + const classified = classifyCredential(request, true); + const lookupPath: BearerLookupPath = + classified.credentialKind === "additional_api_key" && + !authFeatureControls.additionalApiKeyLookupEnabled() + ? "additional_skipped" + : classified.lookupPath; + let result: ApiAuthResult = "error"; + + try { + const value = await operation(); + result = value?.ok ? "success" : lookupPath === "additional_skipped" ? "disabled" : "invalid"; + recordAuthAttempt("legacy", classified.credentialKind, lookupPath, result); + return value; + } catch (error) { + recordAuthAttempt("legacy", classified.credentialKind, lookupPath, result); + throw error; + } finally { + telemetry.duration.record(performance.now() - startedAt, { + resolver: "legacy", + credential_kind: classified.credentialKind, + result, + lookup_path: lookupPath, + }); + } +} + +function recordAuthAttempt( + resolver: "rbac" | "legacy", + credentialKind: BearerCredentialKind, + lookupPath: BearerLookupPath, + result: ApiAuthResult +) { + telemetry.attempts.add(1, { + resolver, + credential_kind: credentialKind, + result, + lookup_path: lookupPath, + }); +} + +// Best-effort pre-classification from the raw token format. This is only used +// for the metric attributes when the resolver throws before returning a +// resolution; the resolver's own resolution is authoritative on success/failure. +// Never records the credential itself — only its bounded format class. +function classifyCredential( + request: Request, + allowJWT: boolean +): { credentialKind: BearerCredentialKind; lookupPath: BearerLookupPath } { + const token = request.headers + .get("Authorization") + ?.replace(/^Bearer /, "") + .trim(); + if (!token) return { credentialKind: "unknown", lookupPath: "not_found" }; + if (token.startsWith("pk_")) { + return { credentialKind: "legacy_public_key", lookupPath: "legacy_public" }; + } + if (allowJWT && isPublicJWT(token)) { + return { credentialKind: "public_jwt", lookupPath: "jwt_current" }; + } + if (isAdditionalApiKey(token)) { + return { credentialKind: "additional_api_key", lookupPath: "additional" }; + } + return token.startsWith("tr_") + ? { credentialKind: "root_api_key", lookupPath: "root_current" } + : { credentialKind: "unknown", lookupPath: "not_found" }; +} diff --git a/apps/webapp/app/services/environmentVariableApiAccess.server.ts b/apps/webapp/app/services/environmentVariableApiAccess.server.ts index 11d401125ac..e9fe2ef781c 100644 --- a/apps/webapp/app/services/environmentVariableApiAccess.server.ts +++ b/apps/webapp/app/services/environmentVariableApiAccess.server.ts @@ -1,10 +1,73 @@ import { json } from "@remix-run/server-runtime"; import type { RuntimeEnvironmentType } from "@trigger.dev/database"; import { isUserActorToken } from "@trigger.dev/rbac"; +import type { RbacAbility } from "@trigger.dev/rbac"; +import { + authenticateApiKeyWithScope, + authenticateRequest, + type AuthenticationResult, +} from "~/services/apiAuth.server"; import { rbac } from "~/services/rbac.server"; type EnvironmentScopedResource = "envvars" | "apiKeys"; +type EnvironmentScopedAuthentication = + | { ok: true; authentication: AuthenticationResult } + | { ok: false; status: 401 | 403; error: string }; + +/** + * Returns the credential already presented by an authenticated API-key caller. + * API-key exchanges must reuse this value instead of exposing the environment's + * root credential. + */ +export function presentedApiKeyFromAuthentication( + authentication: AuthenticationResult +): string | undefined { + return authentication.type === "apiKey" && authentication.result.ok + ? authentication.result.apiKey + : undefined; +} + +/** + * Keep PAT/OAT authentication on the legacy path while routing machine API + * keys through the RBAC controller, where plugin grants are applied. + */ +export async function authenticateEnvironmentScopedApiRequest( + request: Request, + action: "read" | "write", + resource: EnvironmentScopedResource +): Promise<EnvironmentScopedAuthentication> { + const userOrOrganizationAuthentication = await authenticateRequest(request, { + personalAccessToken: true, + organizationAccessToken: true, + apiKey: false, + }); + if (userOrOrganizationAuthentication) { + return { ok: true, authentication: userOrOrganizationAuthentication }; + } + + const apiKeyAuthentication = await authenticateApiKeyWithScope(request, { + action, + resource: { type: resource }, + }); + if (!apiKeyAuthentication.ok) { + return apiKeyAuthentication; + } + + return { + ok: true, + authentication: { type: "apiKey", result: apiKeyAuthentication.authentication }, + }; +} + +/** Env var API routes: PAT/OAT on the legacy path, machine keys via RBAC. */ +export function authenticateEnvVarApiRequest( + request: Request, + action: "read" | "write" +): Promise<EnvironmentScopedAuthentication> { + return authenticateEnvironmentScopedApiRequest(request, action, "envvars"); +} + const RESOURCE_LABELS: Record<EnvironmentScopedResource, string> = { envvars: "environment variables", apiKeys: "API keys", @@ -14,8 +77,8 @@ const RESOURCE_LABELS: Record<EnvironmentScopedResource, string> = { * Env-tier RBAC for environment-scoped API routes (env vars, and the endpoints * that hand out an environment's secret credentials). * - * Machine credentials (an environment's secret/public API key) are already - * scoped to a single environment, so they pass through unchanged. A personal + * Machine credentials (an environment's API key) are authorized by the + * ability returned by the RBAC bearer controller. A personal * access token (or a delegated user-actor token) carries a user, so enforce * that user's role for the targeted environment tier — e.g. a Developer can't * read deployed env vars or API keys via the API, matching the dashboard @@ -34,6 +97,7 @@ export async function authorizePatEnvironmentAccess({ envType, resource, action, + ability, }: { request: Request; authType: "personalAccessToken" | "organizationAccessToken" | "apiKey"; @@ -42,6 +106,8 @@ export async function authorizePatEnvironmentAccess({ envType: RuntimeEnvironmentType; resource: EnvironmentScopedResource; action: "read" | "write"; + // Controller ability for API-key credentials. Absent for PAT/OAT callers. + ability?: RbacAbility; }): Promise<Response | undefined> { const bearer = request.headers .get("Authorization") @@ -49,8 +115,22 @@ export async function authorizePatEnvironmentAccess({ .trim(); const isUat = !!bearer && isUserActorToken(bearer); - // Machine creds (apiKey) and org tokens carry no user role to enforce. A - // user-actor token carries a user just like a PAT, so it's gated too. + // Machine API keys are authorized by their controller ability. Root keys and + // ungranted additional keys are permissive; granted keys are restricted. + if (authType === "apiKey") { + if (ability?.can(action, { type: resource })) { + return undefined; + } + return json( + { + error: `You don't have permission to access this environment's ${RESOURCE_LABELS[resource]}.`, + }, + { status: 403 } + ); + } + + // Org tokens carry no user role to enforce. A user-actor token carries a + // user just like a PAT, so it's gated too. if (authType !== "personalAccessToken" && !isUat) { return undefined; } @@ -82,6 +162,7 @@ export function authorizeEnvVarApiRequest(opts: { projectId: string; envType: RuntimeEnvironmentType; action: "read" | "write"; + ability?: RbacAbility; }): Promise<Response | undefined> { return authorizePatEnvironmentAccess({ ...opts, resource: "envvars" }); } diff --git a/apps/webapp/app/services/publicAccessTokenResponse.server.ts b/apps/webapp/app/services/publicAccessTokenResponse.server.ts new file mode 100644 index 00000000000..f4f20bcf7d8 --- /dev/null +++ b/apps/webapp/app/services/publicAccessTokenResponse.server.ts @@ -0,0 +1,33 @@ +import { generateJWT } from "@trigger.dev/core/v3"; +import { resolveJwtSigningKey } from "@trigger.dev/rbac"; + +export type PublicAccessTokenEnvironment = { + id: string; + apiKey: string; + parentEnvironment?: { apiKey: string } | null; +}; + +export async function publicAccessTokenResponseHeaders({ + environment, + scopes, + expirationTime, +}: { + environment: PublicAccessTokenEnvironment; + scopes: string[]; + expirationTime: string; +}): Promise<Record<string, string>> { + const jwt = await generateJWT({ + secretKey: resolveJwtSigningKey(environment), + payload: { + sub: environment.id, + pub: true, + scopes, + }, + expirationTime, + }); + + return { + "x-trigger-jwt-claims": JSON.stringify({ sub: environment.id, pub: true }), + "x-trigger-jwt": jwt, + }; +} diff --git a/apps/webapp/app/services/rbac.server.ts b/apps/webapp/app/services/rbac.server.ts index 11510c1358f..fc1cfd58cf1 100644 --- a/apps/webapp/app/services/rbac.server.ts +++ b/apps/webapp/app/services/rbac.server.ts @@ -2,6 +2,7 @@ import { $replica, prisma } from "~/db.server"; import type { PrismaClient } from "@trigger.dev/database"; import plugin from "@trigger.dev/rbac"; import { env } from "~/env.server"; +import { authFeatureControls } from "~/services/authFeatureControls.server"; // plugin.create() is synchronous — returns a lazy controller that resolves // any installed RBAC plugin on first call. Top-level await is not used @@ -30,6 +31,7 @@ export const rbac = plugin.create( { forceFallback: env.RBAC_FORCE_FALLBACK, userActorSecret: env.SESSION_SECRET, + additionalApiKeyLookupEnabled: authFeatureControls.additionalApiKeyLookupEnabled, // A plugin that owns its own database client gets the same // writer/replica topology the webapp's Prisma clients use (see // getClient/getReplicaClient in db.server.ts): control-plane URLs win, diff --git a/apps/webapp/app/services/realtime/jwtAuth.server.ts b/apps/webapp/app/services/realtime/jwtAuth.server.ts index ed991214309..2806a737017 100644 --- a/apps/webapp/app/services/realtime/jwtAuth.server.ts +++ b/apps/webapp/app/services/realtime/jwtAuth.server.ts @@ -1,4 +1,10 @@ -import { validateJWT, type ValidationResult } from "@trigger.dev/core/v3/jwt"; +import { + extractJWTSub, + isPublicJWT, + validateJWT, + type ValidationResult, +} from "@trigger.dev/core/v3/jwt"; +import { resolveJwtSigningKey } from "@trigger.dev/rbac"; import { $replica } from "~/db.server"; import { findEnvironmentById } from "~/models/runtimeEnvironment.server"; import type { AuthenticatedEnvironment } from "../apiAuth.server"; @@ -33,10 +39,10 @@ export async function validatePublicJwtKey(token: string): Promise<ValidatePubli return { ok: false, error: "Invalid Public Access Token, environment not found." }; } - let result = await validateJWT( - token, - environment.parentEnvironment?.apiKey ?? environment.apiKey - ); + // A disabled root key does not invalidate public JWTs: disabling rotates + // the stored apiKey (killing tokens signed with the old value), and the + // rotated value keeps signing server-issued tokens so Realtime still works. + let result = await validateJWT(token, resolveJwtSigningKey(environment)); // PATs are signed with the env's apiKey at mint time. If the env's apiKey // has since been rotated, signature verification fails against the current @@ -105,60 +111,8 @@ async function validateAgainstRevokedApiKeys( return primaryResult; } -export function isPublicJWT(token: string): boolean { - // Split the token - const parts = token.split("."); - if (parts.length !== 3) return false; - - try { - // Decode the payload (second part) - const payload = JSON.parse(decodeBase64Url(parts[1])); - - if (payload === null || typeof payload !== "object") return false; - - // Check for the pub: true claim - return "pub" in payload && payload.pub === true; - } catch (_error) { - // If there's any error in decoding or parsing, it's not a valid JWT - return false; - } -} +export { isPublicJWT }; export function extractJwtSigningSecretKey(environment: AuthenticatedEnvironment) { - return environment.parentEnvironment?.apiKey ?? environment.apiKey; -} - -function extractJWTSub(token: string): string | undefined { - // Split the token - const parts = token.split("."); - if (parts.length !== 3) return; - - try { - // Decode the payload (second part) - const payload = JSON.parse(decodeBase64Url(parts[1])); - - if (payload === null || typeof payload !== "object") return; - - // Check for the pub: true claim - return "sub" in payload && typeof payload.sub === "string" ? payload.sub : undefined; - } catch (_error) { - // If there's any error in decoding or parsing, it's not a valid JWT - return; - } -} - -function decodeBase64Url(str: string): string { - // Replace URL-safe characters and add padding - str = str.replace(/-/g, "+").replace(/_/g, "/"); - switch (str.length % 4) { - case 2: - str += "=="; - break; - case 3: - str += "="; - break; - } - - // Decode using Node.js Buffer - return Buffer.from(str, "base64").toString("utf8"); + return resolveJwtSigningKey(environment); } diff --git a/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts b/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts index 94c2929f5f7..83212a6e8b2 100644 --- a/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts +++ b/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts @@ -6,6 +6,7 @@ import { fromZodError } from "zod-validation-error"; import { apiCors } from "~/utils/apiCors"; import { logger } from "../logger.server"; import { rbac } from "../rbac.server"; +import { authenticateBearerWithTelemetry } from "~/services/authTelemetry.server"; import type { RbacAbility, RbacResource } from "@trigger.dev/rbac"; import { isUserActorToken } from "@trigger.dev/rbac"; import type { PersonalAccessTokenAuthenticationResult } from "../personalAccessToken.server"; @@ -53,9 +54,14 @@ async function authenticateRequestForApiBuilder( { allowJWT }: { allowJWT: boolean } ): Promise< | { ok: false; status: 401 | 403; error: string } - | { ok: true; authentication: ApiAuthenticationResultSuccess; ability: RbacAbility } + | { + ok: true; + authentication: ApiAuthenticationResultSuccess; + ability: RbacAbility; + restrictedApiKey: boolean; + } > { - const result = await rbac.authenticateBearer(request, { allowJWT }); + const result = await authenticateBearerWithTelemetry(request, { allowJWT }); if (!result.ok) { // Plugin auth distinguishes 401 (who are you?) from 403 (you're not // allowed) — e.g. a suspended account or IP block returns 403. @@ -78,7 +84,47 @@ async function authenticateRequestForApiBuilder( actor: result.jwt?.act, }; - return { ok: true, authentication, ability: result.ability }; + return { + ok: true, + authentication, + ability: result.ability, + restrictedApiKey: result.subject.type === "apiKey" && result.subject.restricted, + }; +} + +export function shouldRejectRestrictedKeyWithoutAuthorization( + restrictedApiKey: boolean, + hasAuthorization: boolean +): boolean { + return restrictedApiKey && !hasAuthorization; +} + +async function rejectRestrictedKeyWithoutAuthorization({ + request, + restrictedApiKey, + hasAuthorization, + useCors, +}: { + request: Request; + restrictedApiKey: boolean; + hasAuthorization: boolean; + useCors: boolean; +}): Promise<Response | undefined> { + if (!shouldRejectRestrictedKeyWithoutAuthorization(restrictedApiKey, hasAuthorization)) return; + + return wrapResponse( + request, + json( + { + error: "Unauthorized", + code: "unauthorized", + param: "access_token", + type: "authorization", + }, + { status: 403 } + ), + useCors + ); } type AnyZodSchema = z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion<any, any>; @@ -115,14 +161,18 @@ type AnyResourceAuth = { type EveryResourceAuth = { readonly [EVERY_RESOURCE_MARKER]: true; readonly resources: readonly RbacResource[]; + readonly orResources: readonly RbacResource[]; }; export function anyResource(resources: RbacResource[]): AnyResourceAuth { return { [ANY_RESOURCE_MARKER]: true, resources }; } -export function everyResource(resources: RbacResource[]): EveryResourceAuth { - return { [EVERY_RESOURCE_MARKER]: true, resources }; +export function everyResource( + resources: RbacResource[], + orResources: RbacResource[] = [] +): EveryResourceAuth { + return { [EVERY_RESOURCE_MARKER]: true, resources, orResources }; } function isAnyResource(value: unknown): value is AnyResourceAuth { @@ -143,8 +193,15 @@ function isEveryResource(value: unknown): value is EveryResourceAuth { type AuthResource = RbacResource | AnyResourceAuth | EveryResourceAuth; -function checkAuth(ability: RbacAbility, action: string, resource: AuthResource): boolean { +export function checkAuth(ability: RbacAbility, action: string, resource: AuthResource): boolean { if (isEveryResource(resource)) { + // A broad collection grant may be supplied as an alternative to all + // instance checks. This preserves scopes such as read:runs while making + // mixed selected-task filters require access to every requested task. + if (resource.orResources.length > 0 && ability.can(action, [...resource.orResources])) { + return true; + } + // Empty array via [].every() is vacuously true — would let any token // pass auth. Routes building everyResource() from request bodies // (e.g. batch trigger items) should never produce zero elements @@ -223,6 +280,7 @@ type ApiKeyHandlerFunction< ? z.infer<THeadersSchema> : undefined; authentication: ApiAuthenticationResultSuccess; + ability: RbacAbility; request: Request; resource: NonNullable<TResource>; apiVersion: API_VERSIONS; @@ -263,6 +321,13 @@ export function createLoaderApiRoute< ); } const { authentication: authenticationResult, ability } = authResult; + const restrictedKeyRejection = await rejectRestrictedKeyWithoutAuthorization({ + request, + restrictedApiKey: authResult.restrictedApiKey, + hasAuthorization: authorization !== undefined, + useCors: corsStrategy !== "none", + }); + if (restrictedKeyRejection) return restrictedKeyRejection; let parsedParams: any = undefined; if (paramsSchema) { @@ -364,6 +429,7 @@ export function createLoaderApiRoute< searchParams: parsedSearchParams, headers: parsedHeaders, authentication: authenticationResult, + ability, request, resource, apiVersion, @@ -986,6 +1052,7 @@ type ApiKeyActionHandlerFunction< ? z.infer<TBodySchema> : undefined; authentication: ApiAuthenticationResultSuccess; + ability: RbacAbility; request: Request; resource?: TResource; }) => Promise<Response>; @@ -1055,6 +1122,13 @@ export function createActionApiRoute< ); } const { authentication: authenticationResult, ability } = authResult; + const restrictedKeyRejection = await rejectRestrictedKeyWithoutAuthorization({ + request, + restrictedApiKey: authResult.restrictedApiKey, + hasAuthorization: authorization !== undefined, + useCors: corsStrategy !== "none", + }); + if (restrictedKeyRejection) return restrictedKeyRejection; if (maxContentLength) { const contentLength = request.headers.get("content-length"); @@ -1212,6 +1286,7 @@ export function createActionApiRoute< headers: parsedHeaders, body: parsedBody, authentication: authenticationResult, + ability, request, resource, }) @@ -1340,6 +1415,13 @@ export function createMultiMethodApiRoute< ); } const { authentication: authenticationResult, ability } = authResult; + const restrictedKeyRejection = await rejectRestrictedKeyWithoutAuthorization({ + request, + restrictedApiKey: authResult.restrictedApiKey, + hasAuthorization: authorization !== undefined, + useCors: corsStrategy !== "none", + }); + if (restrictedKeyRejection) return restrictedKeyRejection; if (maxContentLength) { const contentLength = request.headers.get("content-length"); diff --git a/apps/webapp/app/utils/batchItemAuthorization.ts b/apps/webapp/app/utils/batchItemAuthorization.ts new file mode 100644 index 00000000000..45b1988877d --- /dev/null +++ b/apps/webapp/app/utils/batchItemAuthorization.ts @@ -0,0 +1,96 @@ +import type { RbacAbility } from "@trigger.dev/rbac"; + +export class BatchItemAuthorizationError extends Error {} + +export function batchPublicAccessScopes( + batchId: string, + ability: RbacAbility, + browserClient: boolean +): string[] { + const scopes = [`read:batch:${batchId}`]; + + if (browserClient && ability.can("batchTrigger", { type: "tasks" })) { + scopes.push(`write:batch:${batchId}`); + } + + return scopes; +} + +/** + * Wraps `authorizeBatchItems` so a credential without a batch-level write grant + * must present at least one authorized item before the batch is touched at all. + * + * A selected-task credential can only be authorized by the items it streams — + * by design, see the phase-two comment in `api.v3.batches.ts`. So a stream that + * yields nothing passes zero checks, and because `StreamBatchItemsService` does + * its batch lookup *before* it ever pulls an item, its "not found" / "not in + * PENDING (current: …)" / `runCount` responses would leak an arbitrary batch's + * existence, lifecycle state and progress to any key in the environment. This + * route bypasses the route builder's `restrictedApiKey && !authorization -> 403` + * fail-closed, so nothing else stops that probe. + * + * Pulling and authorizing the first item up front closes it: no item, no + * authorization. The auth layer must never grant on no input — the same rule + * `checkAuth` applies for empty resource arrays in `apiBuilder.server.ts`. + */ +export async function authorizedBatchItemStream( + items: AsyncIterable<unknown>, + ability: RbacAbility, + batchId: string +): Promise<AsyncIterable<unknown>> { + const authorized = authorizeBatchItems(items, ability, batchId); + + // A batch-level write grant is authorization in its own right, so an empty + // stream stays legal for credentials that own the batch: root keys (via the + // permissive ability) and phase one's delegated browser token. Only a + // restricted credential, which has nothing but its items to go on, is held + // to the at-least-one-item rule. + if (ability.can("write", { type: "batch", id: batchId })) { + return authorized; + } + + const iterator = authorized[Symbol.asyncIterator](); + const first = await iterator.next(); + + if (first.done) { + throw new BatchItemAuthorizationError(); + } + + return (async function* () { + try { + yield first.value; + + while (true) { + const next = await iterator.next(); + if (next.done) return; + yield next.value; + } + } finally { + // Consumers can stop early — p-map abandons the iterator when a mapper + // rejects. Close the generator we pulled from so it unwinds its own + // `for await` and releases the request body stream. + await iterator.return?.(); + } + })(); +} + +export async function* authorizeBatchItems( + items: AsyncIterable<unknown>, + ability: RbacAbility, + batchId: string +): AsyncIterable<unknown> { + const canWriteBatch = ability.can("write", { type: "batch", id: batchId }); + + for await (const item of items) { + const task = + typeof item === "object" && item !== null && "task" in item && typeof item.task === "string" + ? item.task + : undefined; + + if (!canWriteBatch && (!task || !ability.can("batchTrigger", { type: "tasks", id: task }))) { + throw new BatchItemAuthorizationError(); + } + + yield item; + } +} diff --git a/apps/webapp/app/utils/parentRunAuthorization.server.ts b/apps/webapp/app/utils/parentRunAuthorization.server.ts new file mode 100644 index 00000000000..fac9b96dd20 --- /dev/null +++ b/apps/webapp/app/utils/parentRunAuthorization.server.ts @@ -0,0 +1,27 @@ +import type { RbacAbility } from "@trigger.dev/rbac"; +import { resolveRunForMutation } from "~/v3/mollifier/resolveRunForMutation.server"; +import { canWriteResolvedParentRun } from "./parentRunAuthorization"; + +export async function canWriteParentRun( + ability: RbacAbility, + environmentId: string, + organizationId: string, + parentRunId: string | null | undefined +): Promise<boolean> { + if (!parentRunId) return true; + + // A type-level `write:runs` grant covers every run in the environment, so + // resolving the parent could only confirm what we already know. Root keys and + // any other unrestricted credential take this branch, which keeps the trigger + // hot path free of the extra lookup. + if (ability.can("write", { type: "runs" })) return true; + + const run = await resolveRunForMutation({ + environmentId, + organizationId, + runParam: parentRunId, + }); + if (!run) return false; + + return canWriteResolvedParentRun(ability, run); +} diff --git a/apps/webapp/app/utils/parentRunAuthorization.ts b/apps/webapp/app/utils/parentRunAuthorization.ts new file mode 100644 index 00000000000..fc245ed2428 --- /dev/null +++ b/apps/webapp/app/utils/parentRunAuthorization.ts @@ -0,0 +1,13 @@ +import type { RbacAbility } from "@trigger.dev/rbac"; + +type ParentRunResource = { + friendlyId: string; + taskIdentifier: string; +}; + +export function canWriteResolvedParentRun(ability: RbacAbility, run: ParentRunResource): boolean { + return ability.can("write", [ + { type: "runs", id: run.friendlyId }, + { type: "tasks", id: run.taskIdentifier }, + ]); +} diff --git a/apps/webapp/app/utils/requestIdempotency.test.ts b/apps/webapp/app/utils/requestIdempotency.test.ts new file mode 100644 index 00000000000..a422646c63a --- /dev/null +++ b/apps/webapp/app/utils/requestIdempotency.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest"; +import { scopeRequestIdempotencyKey } from "./requestIdempotencyKey"; + +describe("scopeRequestIdempotencyKey", () => { + it("keeps retries stable within the same environment and task scope", () => { + expect(scopeRequestIdempotencyKey("request-1", ["env-1", "task-a"])).toBe( + scopeRequestIdempotencyKey("request-1", ["env-1", "task-a"]) + ); + }); + + it("does not share cache entries across environments or tasks", () => { + const original = scopeRequestIdempotencyKey("request-1", ["env-1", "task-a"]); + + expect(scopeRequestIdempotencyKey("request-1", ["env-1", "task-b"])).not.toBe(original); + expect(scopeRequestIdempotencyKey("request-1", ["env-2", "task-a"])).not.toBe(original); + }); + + it("preserves missing request keys", () => { + expect(scopeRequestIdempotencyKey(undefined, ["env-1", "task-a"])).toBeUndefined(); + expect(scopeRequestIdempotencyKey(null, ["env-1", "task-a"])).toBeNull(); + }); +}); diff --git a/apps/webapp/app/utils/requestIdempotencyKey.ts b/apps/webapp/app/utils/requestIdempotencyKey.ts new file mode 100644 index 00000000000..90594f1a0ba --- /dev/null +++ b/apps/webapp/app/utils/requestIdempotencyKey.ts @@ -0,0 +1,12 @@ +import { createHash } from "node:crypto"; + +export function scopeRequestIdempotencyKey( + requestIdempotencyKey: string | null | undefined, + scope: readonly string[] +): string | null | undefined { + if (!requestIdempotencyKey) return requestIdempotencyKey; + + return createHash("sha256") + .update(JSON.stringify([requestIdempotencyKey, ...scope])) + .digest("hex"); +} diff --git a/apps/webapp/app/v3/featureFlags.ts b/apps/webapp/app/v3/featureFlags.ts index 78a919b20ba..74074214812 100644 --- a/apps/webapp/app/v3/featureFlags.ts +++ b/apps/webapp/app/v3/featureFlags.ts @@ -24,6 +24,9 @@ export const FEATURE_FLAG = { runOpsMintKindPrev: "runOpsMintKindPrev", runOpsMintKindFlippedAt: "runOpsMintKindFlippedAt", queueMetricsUiEnabled: "queueMetricsUiEnabled", + // System-wide kill switch for additional (scoped) environment API-key lookup. + // Defaults off; enable during rollout once the new lookup path is trusted. + additionalApiKeyLookupEnabled: "additionalApiKeyLookupEnabled", } as const; export const FeatureFlagCatalog = { @@ -72,6 +75,10 @@ export const FeatureFlagCatalog = { // Per-org access to the Queue Metrics dashboard UI (view only; emission is global and // separate). Off unless enabled for the org. [FEATURE_FLAG.queueMetricsUiEnabled]: z.coerce.boolean(), + // Strict z.boolean() (not z.coerce.boolean()): coercion turns the string + // "false" into true, which would silently enable this kill switch the wrong + // way if written as a string. Cold/absent resolves to the safe `false`. + [FEATURE_FLAG.additionalApiKeyLookupEnabled]: z.boolean(), }; export type FeatureFlagKey = keyof typeof FeatureFlagCatalog; @@ -90,6 +97,8 @@ export const ORG_LOCKED_FLAGS: FeatureFlagKey[] = [ FEATURE_FLAG.taskEventRepository, FEATURE_FLAG.runOpsMintKindPrev, FEATURE_FLAG.runOpsMintKindFlippedAt, + // System-wide only — an org must not be able to override the rollout switch. + FEATURE_FLAG.additionalApiKeyLookupEnabled, ]; // Create a Zod schema from the existing catalog diff --git a/apps/webapp/app/v3/mollifier/resolveRunForMutation.server.ts b/apps/webapp/app/v3/mollifier/resolveRunForMutation.server.ts index 258c0da2117..4679ef91b77 100644 --- a/apps/webapp/app/v3/mollifier/resolveRunForMutation.server.ts +++ b/apps/webapp/app/v3/mollifier/resolveRunForMutation.server.ts @@ -15,8 +15,8 @@ import { getMollifierBuffer as defaultGetBuffer } from "./mollifierBuffer.server // `findResource: async () => null`, which made every cancel 404 before // the action ran. The helper makes the lookup unit-testable.) export type ResolvedRunForMutation = - | { source: "pg"; friendlyId: string } - | { source: "buffer"; friendlyId: string }; + | { source: "pg"; friendlyId: string; taskIdentifier: string } + | { source: "buffer"; friendlyId: string; taskIdentifier: string }; export type ResolveRunForMutationDeps = { prismaReplica?: PrismaReplicaClient; @@ -36,17 +36,34 @@ export async function resolveRunForMutation(input: { const pgRun = await runStore.findRun( { friendlyId: input.runParam, runtimeEnvironmentId: input.environmentId }, - { select: { friendlyId: true } }, + { select: { friendlyId: true, taskIdentifier: true } }, replica ); - if (pgRun) return { source: "pg", friendlyId: pgRun.friendlyId }; + if (pgRun) { + return { + source: "pg", + friendlyId: pgRun.friendlyId, + taskIdentifier: pgRun.taskIdentifier, + }; + } const buffer = getBuffer(); if (buffer) { const entry = await buffer.getEntry(input.runParam); if (entry && entry.envId === input.environmentId && entry.orgId === input.organizationId) { - return { source: "buffer", friendlyId: input.runParam }; + try { + const snapshot = JSON.parse(entry.payload) as { taskIdentifier?: unknown }; + if (typeof snapshot.taskIdentifier === "string") { + return { + source: "buffer", + friendlyId: input.runParam, + taskIdentifier: snapshot.taskIdentifier, + }; + } + } catch { + // A malformed snapshot is not an authorizable run resource. + } } } @@ -64,10 +81,16 @@ export async function resolveRunForMutation(input: { // downstream mutateWithFallback flow would otherwise handle correctly. const writerRun = await runStore.findRun( { friendlyId: input.runParam, runtimeEnvironmentId: input.environmentId }, - { select: { friendlyId: true } }, + { select: { friendlyId: true, taskIdentifier: true } }, writer ); - if (writerRun) return { source: "pg", friendlyId: writerRun.friendlyId }; + if (writerRun) { + return { + source: "pg", + friendlyId: writerRun.friendlyId, + taskIdentifier: writerRun.taskIdentifier, + }; + } return null; } diff --git a/apps/webapp/test/apiAuthScope.test.ts b/apps/webapp/test/apiAuthScope.test.ts new file mode 100644 index 00000000000..ab3d12c273c --- /dev/null +++ b/apps/webapp/test/apiAuthScope.test.ts @@ -0,0 +1,163 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const rbacMocks = vi.hoisted(() => ({ + authenticateBearer: vi.fn<(...args: any[]) => Promise<any>>(), +})); + +const telemetryMocks = vi.hoisted(() => ({ + attemptsAdd: vi.fn(), + durationRecord: vi.fn(), +})); + +vi.mock("@internal/tracing", () => ({ + getMeter: () => ({ + createCounter: () => ({ add: telemetryMocks.attemptsAdd }), + createHistogram: () => ({ record: telemetryMocks.durationRecord }), + createObservableGauge: () => ({ addCallback: vi.fn() }), + }), +})); +vi.mock("~/services/rbac.server", () => ({ rbac: rbacMocks })); +vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} })); +vi.mock("~/env.server", () => ({ env: { SESSION_SECRET: "test-session-secret" } })); +vi.mock("~/models/project.server", () => ({ findProjectByRef: vi.fn() })); +vi.mock("~/models/runtimeEnvironment.server", () => ({ + authIncludeBase: {}, + authIncludeWithParent: {}, + findEnvironmentByApiKey: vi.fn(), + findEnvironmentByPublicApiKey: vi.fn(), + toAuthenticated: vi.fn(), +})); +vi.mock("~/services/personalAccessToken.server", () => ({ + authenticateApiRequestWithPersonalAccessToken: vi.fn(), + isPersonalAccessToken: () => false, +})); +vi.mock("~/services/organizationAccessToken.server", () => ({ + authenticateApiRequestWithOrganizationAccessToken: vi.fn(), + isOrganizationAccessToken: () => false, +})); +vi.mock("~/services/realtime/jwtAuth.server", () => ({ + isPublicJWT: () => false, + validatePublicJwtKey: vi.fn(), +})); +vi.mock("~/services/logger.server", () => ({ + logger: { debug: vi.fn(), error: vi.fn(), warn: vi.fn() }, +})); + +import { authenticateApiKeyWithScope } from "~/services/apiAuth.server"; + +describe("authenticateApiKeyWithScope", () => { + beforeEach(() => { + rbacMocks.authenticateBearer.mockReset(); + telemetryMocks.attemptsAdd.mockReset(); + telemetryMocks.durationRecord.mockReset(); + }); + + it("returns 401 without a bearer credential", async () => { + const result = await authenticateApiKeyWithScope(new Request("https://example.com"), { + action: "read", + resource: { type: "envvars" }, + }); + + expect(result).toEqual({ + ok: false, + status: 401, + error: "Invalid or Missing API key", + }); + expect(rbacMocks.authenticateBearer).not.toHaveBeenCalled(); + }); + + it.each([ + { status: 401 as const, error: "Invalid API key" }, + { status: 403 as const, error: "Unauthorized" }, + ])("preserves controller $status failures", async (failure) => { + rbacMocks.authenticateBearer.mockResolvedValue({ ok: false, ...failure }); + const request = new Request("https://example.com", { + headers: { Authorization: "Bearer tr_test_key" }, + }); + + await expect( + authenticateApiKeyWithScope(request, { + action: "write", + resource: { type: "deployments" }, + }) + ).resolves.toEqual({ ok: false, ...failure }); + }); + + it("bridges controller success into the legacy private authentication shape", async () => { + const environment = { id: "env_123" }; + const ability = { can: vi.fn(() => true), canSuper: vi.fn(() => true) }; + rbacMocks.authenticateBearer.mockResolvedValue({ + ok: true, + environment, + ability, + subject: { type: "apiKey", apiKeyId: "key_123" }, + resolution: { credentialKind: "root_api_key", lookupPath: "root_current" }, + }); + const request = new Request("https://example.com", { + headers: { Authorization: "Bearer tr_test_key", "x-trigger-branch": "feature/test" }, + }); + + const result = await authenticateApiKeyWithScope(request, { + action: "read", + resource: { type: "envvars" }, + allowJWT: true, + }); + + expect(rbacMocks.authenticateBearer).toHaveBeenCalledWith(request, { allowJWT: true }); + expect(ability.can).toHaveBeenCalledWith("read", { type: "envvars" }); + expect(telemetryMocks.attemptsAdd).toHaveBeenCalledWith(1, { + resolver: "rbac", + credential_kind: "root_api_key", + result: "success", + lookup_path: "root_current", + }); + expect(telemetryMocks.durationRecord).toHaveBeenCalledWith( + expect.any(Number), + expect.objectContaining({ + resolver: "rbac", + credential_kind: "root_api_key", + result: "success", + lookup_path: "root_current", + }) + ); + expect(result).toEqual({ + ok: true, + authentication: { + ok: true, + apiKey: "tr_test_key", + type: "PRIVATE", + environment, + ability, + }, + }); + }); + + it("records successful authentication before returning an authorization failure", async () => { + const ability = { can: vi.fn(() => false), canSuper: vi.fn(() => false) }; + rbacMocks.authenticateBearer.mockResolvedValue({ + ok: true, + environment: { id: "env_123" }, + ability, + subject: { type: "apiKey", apiKeyId: "key_123" }, + resolution: { credentialKind: "additional_api_key", lookupPath: "additional" }, + }); + const request = new Request("https://example.com", { + headers: { Authorization: "Bearer tr_prod_sk_0123456789abcdefghijklmn" }, + }); + + await expect( + authenticateApiKeyWithScope(request, { + action: "write", + resource: { type: "deployments" }, + }) + ).resolves.toEqual({ ok: false, status: 403, error: "Unauthorized" }); + + expect(ability.can).toHaveBeenCalledWith("write", { type: "deployments" }); + expect(telemetryMocks.attemptsAdd).toHaveBeenCalledWith(1, { + resolver: "rbac", + credential_kind: "additional_api_key", + result: "success", + lookup_path: "additional", + }); + }); +}); diff --git a/apps/webapp/test/apiBuilderAuthorization.test.ts b/apps/webapp/test/apiBuilderAuthorization.test.ts new file mode 100644 index 00000000000..be108bf2794 --- /dev/null +++ b/apps/webapp/test/apiBuilderAuthorization.test.ts @@ -0,0 +1,71 @@ +import { buildJwtAbility } from "@trigger.dev/plugins"; +import { describe, expect, it } from "vitest"; +import { + checkAuth, + everyResource, + shouldRejectRestrictedKeyWithoutAuthorization, +} from "~/services/routeBuilders/apiBuilder.server"; + +describe("restricted API key route authorization", () => { + it("fails closed when a route has no authorization declaration", () => { + expect(shouldRejectRestrictedKeyWithoutAuthorization(true, false)).toBe(true); + expect(shouldRejectRestrictedKeyWithoutAuthorization(true, true)).toBe(false); + expect(shouldRejectRestrictedKeyWithoutAuthorization(false, false)).toBe(false); + }); +}); + +describe("everyResource authorization", () => { + it("requires an ID-scoped ability to match every requested resource", () => { + const ability = buildJwtAbility(["read:tasks:task-a"]); + + expect( + checkAuth( + ability, + "read", + everyResource( + [ + { type: "tasks", id: "task-a" }, + { type: "tasks", id: "task-b" }, + ], + [{ type: "runs" }, { type: "tasks" }] + ) + ) + ).toBe(false); + }); + + it("allows an ID-scoped ability when every requested resource matches", () => { + const ability = buildJwtAbility(["read:tasks:task-a", "read:tasks:task-b"]); + + expect( + checkAuth( + ability, + "read", + everyResource( + [ + { type: "tasks", id: "task-a" }, + { type: "tasks", id: "task-b" }, + ], + [{ type: "runs" }, { type: "tasks" }] + ) + ) + ).toBe(true); + }); + + it("preserves broad collection grants as an alternative", () => { + const ability = buildJwtAbility(["read:runs"]); + + expect( + checkAuth( + ability, + "read", + everyResource( + [ + { type: "tasks", id: "task-a" }, + { type: "tasks", id: "task-b" }, + ], + [{ type: "runs" }, { type: "tasks" }] + ) + ) + ).toBe(true); + }); +}); diff --git a/apps/webapp/test/apiRetrieveRunPresenter.readroute.test.ts b/apps/webapp/test/apiRetrieveRunPresenter.readroute.test.ts index a46cd0894f6..4c6ea3b8082 100644 --- a/apps/webapp/test/apiRetrieveRunPresenter.readroute.test.ts +++ b/apps/webapp/test/apiRetrieveRunPresenter.readroute.test.ts @@ -304,6 +304,7 @@ async function seedRunWithTree( runFriendlyId: `run_${runId}`, parentFriendlyId: `run_${parentId}`, rootFriendlyId: `run_${rootId}`, + childId, childFriendlyId: `run_${childId}`, attemptId: attempt.id, }; @@ -354,6 +355,10 @@ describe("ApiRetrieveRunPresenter.findRun store-routed read (single-DB invariant where: { id: tree.run.id }, data: { scheduleId }, }); + await prisma.taskRun.update({ + where: { id: tree.childId }, + data: { taskIdentifier: "other-task" }, + }); const env = authEnv(organization, project, runtimeEnvironment); const found = await readFoundRunViaStore(store, tree.runFriendlyId, env.id); diff --git a/apps/webapp/test/auth-api.e2e.full.test.ts b/apps/webapp/test/auth-api.e2e.full.test.ts index 988a8f7071c..159eb026467 100644 --- a/apps/webapp/test/auth-api.e2e.full.test.ts +++ b/apps/webapp/test/auth-api.e2e.full.test.ts @@ -744,13 +744,16 @@ describe("API", () => { }); }); - // v3 batches use a collection-level resource { type: "tasks" } with - // no id — items are validated per-row when streamed. So id-specific - // scopes (write:tasks:foo) shouldn't grant blanket access; only - // type-level write:tasks (or admin/write:all) should. - describe("Trigger task — batch v3 (api.v3.batches) collection-level", () => { + // v3 batch creation accepts an optional declaration of the distinct task + // identifiers that will be streamed. New clients send it so selected-task + // credentials can authorize the batch before its shell is created. Older + // clients omit it and retain the collection-level, fail-closed behavior. + describe("Trigger task — batch v3 (api.v3.batches)", () => { const path = "/api/v3/batches"; - const buildBody = () => ({ runCount: 1 }); + const buildBody = (taskIdentifiers?: string[]) => ({ + runCount: taskIdentifiers?.length ?? 1, + taskIdentifiers, + }); it("missing auth: 401", async () => { const server = getTestServer(); @@ -779,6 +782,67 @@ describe("API", () => { expect(res.status).not.toBe(403); }); + it("JWT with batchTrigger:tasks:taskA + declared taskA: auth passes", async () => { + const server = getTestServer(); + const seed = await seedTestEnvironment(server.prisma); + const jwt = await generateJWT({ + secretKey: seed.apiKey, + payload: { + pub: true, + sub: seed.environment.id, + scopes: ["batchTrigger:tasks:taskA"], + }, + expirationTime: "15m", + }); + const res = await server.webapp.fetch(path, { + method: "POST", + headers: { Authorization: `Bearer ${jwt}`, "Content-Type": "application/json" }, + body: JSON.stringify(buildBody(["taskA"])), + }); + expect(res.status).not.toBe(401); + expect(res.status).not.toBe(403); + }); + + it("JWT with batchTrigger:tasks:taskA + a mixed declaration: 403", async () => { + const server = getTestServer(); + const seed = await seedTestEnvironment(server.prisma); + const jwt = await generateJWT({ + secretKey: seed.apiKey, + payload: { + pub: true, + sub: seed.environment.id, + scopes: ["batchTrigger:tasks:taskA"], + }, + expirationTime: "15m", + }); + const res = await server.webapp.fetch(path, { + method: "POST", + headers: { Authorization: `Bearer ${jwt}`, "Content-Type": "application/json" }, + body: JSON.stringify(buildBody(["taskA", "taskB"])), + }); + expect(res.status).toBe(403); + }); + + it("JWT with batchTrigger:tasks:taskA + no declaration: 403", async () => { + const server = getTestServer(); + const seed = await seedTestEnvironment(server.prisma); + const jwt = await generateJWT({ + secretKey: seed.apiKey, + payload: { + pub: true, + sub: seed.environment.id, + scopes: ["batchTrigger:tasks:taskA"], + }, + expirationTime: "15m", + }); + const res = await server.webapp.fetch(path, { + method: "POST", + headers: { Authorization: `Bearer ${jwt}`, "Content-Type": "application/json" }, + body: JSON.stringify(buildBody()), + }); + expect(res.status).toBe(403); + }); + it("JWT with read:tasks: 403", async () => { const server = getTestServer(); const seed = await seedTestEnvironment(server.prisma); @@ -913,7 +977,7 @@ describe("API", () => { expect(res.status).toBe(403); }); - it("filter[taskIdentifier]=task_a,task_b + JWT read:tasks:task_a → passes (array match)", async () => { + it("filter[taskIdentifier]=task_a,task_b + JWT read:tasks:task_a → 403 (requires every task)", async () => { const server = getTestServer(); const seed = await seedTestEnvironment(server.prisma); const jwt = await generateJWT({ @@ -928,11 +992,9 @@ describe("API", () => { const res = await get("?filter%5BtaskIdentifier%5D=task_a%2Ctask_b", { Authorization: `Bearer ${jwt}`, }); - // Resource array is [{type:"runs"}, {type:"tasks",id:"task_a"}, {type:"tasks",id:"task_b"}]. - // The scope read:tasks:task_a matches the second element → access granted. - // Handler may 500 (ClickHouse unreachable in tests) but auth passed. - expect(res.status).not.toBe(401); - expect(res.status).not.toBe(403); + // A task-scoped JWT must authorize every requested task so including an + // unauthorized task in a multi-task filter cannot expose its runs. + expect(res.status).toBe(403); }); it("filter[taskIdentifier]=task_a + JWT read:tasks:task_z → 403 (no array match)", async () => { @@ -2473,13 +2535,14 @@ describe("API", () => { expect(res.status).not.toBe(403); }); - it("read:tasks (type-only) on no-filter list: 403 (filter is sessions, not tasks)", async () => { - // No filter → resource is `{ type: "sessions" }` only. read:tasks - // doesn't match the sessions type, so 403 — explicit narrowing. + it("read:tasks (type-only) on no-filter list: auth passes", async () => { + // Preserve the legacy behavior where a type-level task scope grants + // access to an unfiltered list while task ID scopes require a filter. const seed = await seedTestEnvironment(getTestServer().prisma); const jwt = await mintJwt(seed.apiKey, seed.environment.id, ["read:tasks"]); const res = await fetchWithJwt(jwt); - expect(res.status).toBe(403); + expect(res.status).not.toBe(401); + expect(res.status).not.toBe(403); }); it("write:tasks:foo (wrong action) on filter=foo: 403", async () => { diff --git a/apps/webapp/test/authFeatureControls.test.ts b/apps/webapp/test/authFeatureControls.test.ts new file mode 100644 index 00000000000..5de10e348ab --- /dev/null +++ b/apps/webapp/test/authFeatureControls.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from "vitest"; +import { resolveAuthFeatureControls } from "~/services/authFeatureControls"; +import { FEATURE_FLAG, FeatureFlagCatalog, ORG_LOCKED_FLAGS } from "~/v3/featureFlags"; + +describe("auth feature controls", () => { + it("uses safe defaults for a cold or missing snapshot", () => { + expect(resolveAuthFeatureControls(undefined)).toEqual({ + additionalApiKeyLookupEnabled: false, + }); + }); + + it("accepts only strict booleans and locks org overrides", () => { + const flag = FEATURE_FLAG.additionalApiKeyLookupEnabled; + expect(FeatureFlagCatalog[flag].safeParse(true).success).toBe(true); + // Strict z.boolean(): the stringified "false" must not coerce to true. + expect(FeatureFlagCatalog[flag].safeParse("false").success).toBe(false); + expect(ORG_LOCKED_FLAGS).toContain(flag); + }); +}); diff --git a/apps/webapp/test/environmentVariableApiAccess.test.ts b/apps/webapp/test/environmentVariableApiAccess.test.ts new file mode 100644 index 00000000000..703df55895a --- /dev/null +++ b/apps/webapp/test/environmentVariableApiAccess.test.ts @@ -0,0 +1,111 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const authMocks = vi.hoisted(() => ({ + authenticateRequest: vi.fn<(...args: any[]) => Promise<any>>(), + authenticateApiKeyWithScope: vi.fn<(...args: any[]) => Promise<any>>(), +})); + +vi.mock("~/services/apiAuth.server", () => authMocks); +vi.mock("~/services/rbac.server", () => ({ + rbac: { authenticatePat: vi.fn(), authenticateUserActor: vi.fn() }, +})); + +import { + authenticateEnvVarApiRequest, + presentedApiKeyFromAuthentication, +} from "~/services/environmentVariableApiAccess.server"; + +describe("presentedApiKeyFromAuthentication", () => { + it("returns the API key that authenticated the request", () => { + expect( + presentedApiKeyFromAuthentication({ + type: "apiKey", + result: { + ok: true, + apiKey: "tr_prod_sk_presented", + type: "PRIVATE", + environment: {} as never, + }, + }) + ).toBe("tr_prod_sk_presented"); + }); + + it("does not exchange user tokens for an API key", () => { + expect( + presentedApiKeyFromAuthentication({ + type: "personalAccessToken", + result: { userId: "user_123" } as never, + }) + ).toBeUndefined(); + }); +}); + +describe("authenticateEnvVarApiRequest", () => { + beforeEach(() => { + authMocks.authenticateRequest.mockReset(); + authMocks.authenticateApiKeyWithScope.mockReset(); + }); + + it.each([ + { type: "personalAccessToken", result: { userId: "user_123" } }, + { type: "organizationAccessToken", result: { organizationId: "org_123" } }, + ])("preserves $type authentication", async (authentication) => { + authMocks.authenticateRequest.mockResolvedValue(authentication); + const request = new Request("https://example.com", { + headers: { Authorization: "Bearer token" }, + }); + + await expect(authenticateEnvVarApiRequest(request, "read")).resolves.toEqual({ + ok: true, + authentication, + }); + expect(authMocks.authenticateRequest).toHaveBeenCalledWith(request, { + personalAccessToken: true, + organizationAccessToken: true, + apiKey: false, + }); + expect(authMocks.authenticateApiKeyWithScope).not.toHaveBeenCalled(); + }); + + it("routes API-key credentials through scoped controller authentication", async () => { + const authentication = { + ok: true, + apiKey: "tr_test_key", + type: "PRIVATE", + environment: { id: "env_123" }, + ability: { can: vi.fn(() => true) }, + }; + authMocks.authenticateRequest.mockResolvedValue(undefined); + authMocks.authenticateApiKeyWithScope.mockResolvedValue({ ok: true, authentication }); + const request = new Request("https://example.com", { + headers: { Authorization: "Bearer tr_test_key" }, + }); + + await expect(authenticateEnvVarApiRequest(request, "write")).resolves.toEqual({ + ok: true, + authentication: { type: "apiKey", result: authentication }, + }); + expect(authMocks.authenticateApiKeyWithScope).toHaveBeenCalledWith(request, { + action: "write", + resource: { type: "envvars" }, + }); + }); + + it("preserves scoped controller failures", async () => { + authMocks.authenticateRequest.mockResolvedValue(undefined); + authMocks.authenticateApiKeyWithScope.mockResolvedValue({ + ok: false, + status: 403, + error: "Unauthorized", + }); + + await expect( + authenticateEnvVarApiRequest( + new Request("https://example.com", { + headers: { Authorization: "Bearer tr_test_key" }, + }), + "read" + ) + ).resolves.toEqual({ ok: false, status: 403, error: "Unauthorized" }); + }); +}); diff --git a/apps/webapp/test/findEnvironmentByApiKey.test.ts b/apps/webapp/test/findEnvironmentByApiKey.test.ts index 5bbc1e97834..c301a02b201 100644 --- a/apps/webapp/test/findEnvironmentByApiKey.test.ts +++ b/apps/webapp/test/findEnvironmentByApiKey.test.ts @@ -1,7 +1,8 @@ import { postgresTest } from "@internal/testcontainers"; import { type PrismaClient } from "@trigger.dev/database"; -import { describe, expect, vi } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { findEnvironmentByApiKey } from "~/models/runtimeEnvironment.server"; +import { generateAdditionalApiKey, hashApiKey } from "~/utils/apiKeys"; import { createTestOrgProjectWithMember, uniqueId } from "./fixtures/environmentVariablesFixtures"; vi.setConfig({ testTimeout: 60_000 }); @@ -161,4 +162,208 @@ describe("findEnvironmentByApiKey — non-branchable", () => { const resolved = await findEnvironmentByApiKey("tr_dev_nonexistent", undefined, prisma); expect(resolved).toBeNull(); }); + + it("queries only the additional-key store for a valid additional-key format", async () => { + const runtimeEnvironmentFind = vi.fn(); + const revokedApiKeyFind = vi.fn(); + const apiKeyFind = vi.fn(async () => null); + const tx = { + runtimeEnvironment: { findFirst: runtimeEnvironmentFind }, + revokedApiKey: { findFirst: revokedApiKeyFind }, + apiKey: { findFirst: apiKeyFind }, + } as unknown as PrismaClient; + + await expect( + findEnvironmentByApiKey("tr_prod_sk_0123456789abcdefghijklmn", undefined, tx, () => true) + ).resolves.toBeNull(); + expect(apiKeyFind).toHaveBeenCalledOnce(); + expect(runtimeEnvironmentFind).not.toHaveBeenCalled(); + expect(revokedApiKeyFind).not.toHaveBeenCalled(); + }); + + it("skips the additional-key store when lookup is disabled", async () => { + const runtimeEnvironmentFind = vi.fn(); + const revokedApiKeyFind = vi.fn(); + const apiKeyFind = vi.fn(); + const tx = { + runtimeEnvironment: { findFirst: runtimeEnvironmentFind }, + revokedApiKey: { findFirst: revokedApiKeyFind }, + apiKey: { findFirst: apiKeyFind }, + } as unknown as PrismaClient; + + await expect( + findEnvironmentByApiKey("tr_prod_sk_0123456789abcdefghijklmn", undefined, tx, () => false) + ).resolves.toBeNull(); + expect(apiKeyFind).not.toHaveBeenCalled(); + expect(runtimeEnvironmentFind).not.toHaveBeenCalled(); + expect(revokedApiKeyFind).not.toHaveBeenCalled(); + }); + + it.each(["tr_prod_ak_0123456789abcdefghijklmn", "tr_prod_sk_too-short"])( + "keeps malformed additional-key formats on the root lookup path: %s", + async (apiKey) => { + const runtimeEnvironmentFind = vi.fn(async () => null); + const revokedApiKeyFind = vi.fn(async () => null); + const apiKeyFind = vi.fn(); + const tx = { + runtimeEnvironment: { findFirst: runtimeEnvironmentFind }, + revokedApiKey: { findFirst: revokedApiKeyFind }, + apiKey: { findFirst: apiKeyFind }, + } as unknown as PrismaClient; + + await expect(findEnvironmentByApiKey(apiKey, undefined, tx)).resolves.toBeNull(); + expect(runtimeEnvironmentFind).toHaveBeenCalledOnce(); + expect(revokedApiKeyFind).toHaveBeenCalledOnce(); + expect(apiKeyFind).not.toHaveBeenCalled(); + } + ); +}); + +describe("findEnvironmentByApiKey — additional and disabled keys", () => { + postgresTest("authenticates an active additional key", async ({ prisma }) => { + const { organization, project, user } = await createTestOrgProjectWithMember(prisma); + const environment = await createEnv(prisma, project.id, organization.id, { + type: "PRODUCTION", + }); + const plaintext = generateAdditionalApiKey("PRODUCTION").apiKey; + + await prisma.apiKey.create({ + data: { + name: "External integration", + keyHash: hashApiKey(plaintext), + lastFour: plaintext.slice(-4), + runtimeEnvironmentId: environment.id, + createdByUserId: user.id, + presetId: null, + scopes: ["admin"], + }, + }); + + const resolved = await findEnvironmentByApiKey(plaintext, undefined, prisma, () => true); + + expect(resolved?.id).toBe(environment.id); + expect(resolved?.apiKey).toBe(environment.apiKey); + }); + + postgresTest( + "rejects restricted scopes on the legacy authentication path", + async ({ prisma }) => { + const { organization, project, user } = await createTestOrgProjectWithMember(prisma); + const environment = await createEnv(prisma, project.id, organization.id, { + type: "PRODUCTION", + }); + const plaintext = generateAdditionalApiKey("PRODUCTION").apiKey; + + await prisma.apiKey.create({ + data: { + name: "Task-scoped", + keyHash: hashApiKey(plaintext), + lastFour: plaintext.slice(-4), + runtimeEnvironmentId: environment.id, + createdByUserId: user.id, + presetId: "TASK_SELECTION_TEST_PRESET", + scopes: ["trigger:tasks"], + }, + }); + + await expect( + findEnvironmentByApiKey(plaintext, undefined, prisma, () => true) + ).resolves.toBeNull(); + } + ); + + postgresTest("rejects an additional key with empty scopes", async ({ prisma }) => { + const { organization, project, user } = await createTestOrgProjectWithMember(prisma); + const environment = await createEnv(prisma, project.id, organization.id, { + type: "PRODUCTION", + }); + const plaintext = generateAdditionalApiKey("PRODUCTION").apiKey; + + await prisma.apiKey.create({ + data: { + name: "Empty policy", + keyHash: hashApiKey(plaintext), + lastFour: plaintext.slice(-4), + runtimeEnvironmentId: environment.id, + createdByUserId: user.id, + presetId: null, + scopes: [], + }, + }); + + await expect( + findEnvironmentByApiKey(plaintext, undefined, prisma, () => true) + ).resolves.toBeNull(); + }); + + postgresTest("rejects revoked and expired additional keys", async ({ prisma }) => { + const { organization, project, user } = await createTestOrgProjectWithMember(prisma); + const environment = await createEnv(prisma, project.id, organization.id, { + type: "PRODUCTION", + }); + const revoked = generateAdditionalApiKey("PRODUCTION").apiKey; + const expired = generateAdditionalApiKey("PRODUCTION").apiKey; + + await prisma.apiKey.createMany({ + data: [ + { + name: "Revoked", + keyHash: hashApiKey(revoked), + lastFour: revoked.slice(-4), + runtimeEnvironmentId: environment.id, + createdByUserId: user.id, + presetId: null, + scopes: ["admin"], + revokedAt: new Date(), + }, + { + name: "Expired", + keyHash: hashApiKey(expired), + lastFour: expired.slice(-4), + runtimeEnvironmentId: environment.id, + createdByUserId: user.id, + presetId: null, + scopes: ["admin"], + expiresAt: new Date(Date.now() - 1_000), + }, + ], + }); + + await expect( + findEnvironmentByApiKey(revoked, undefined, prisma, () => true) + ).resolves.toBeNull(); + await expect( + findEnvironmentByApiKey(expired, undefined, prisma, () => true) + ).resolves.toBeNull(); + }); + + postgresTest( + "resolves the same environment from the root key and an additional key", + async ({ prisma }) => { + const { organization, project, user } = await createTestOrgProjectWithMember(prisma); + const environment = await createEnv(prisma, project.id, organization.id, { + type: "PRODUCTION", + }); + const additional = generateAdditionalApiKey("PRODUCTION").apiKey; + + await prisma.apiKey.create({ + data: { + name: "Replacement", + keyHash: hashApiKey(additional), + lastFour: additional.slice(-4), + runtimeEnvironmentId: environment.id, + createdByUserId: user.id, + presetId: null, + scopes: ["admin"], + }, + }); + + await expect( + findEnvironmentByApiKey(environment.apiKey, undefined, prisma) + ).resolves.toMatchObject({ id: environment.id }); + await expect( + findEnvironmentByApiKey(additional, undefined, prisma, () => true) + ).resolves.toMatchObject({ id: environment.id }); + } + ); }); diff --git a/apps/webapp/test/mollifierResolveRunForMutation.test.ts b/apps/webapp/test/mollifierResolveRunForMutation.test.ts index b50d8ad9400..0b18095319c 100644 --- a/apps/webapp/test/mollifierResolveRunForMutation.test.ts +++ b/apps/webapp/test/mollifierResolveRunForMutation.test.ts @@ -19,11 +19,13 @@ import type { BufferEntry, MollifierBuffer } from "@trigger.dev/redis-worker"; const NOW = new Date("2026-05-21T10:00:00Z"); -function fakeReplica(row: { friendlyId: string } | null) { - return { taskRun: { findFirst: vi.fn(async () => row) } }; +function fakeReplica(row: { friendlyId: string; taskIdentifier?: string } | null) { + const result = row ? { ...row, taskIdentifier: row.taskIdentifier ?? "task-1" } : null; + return { taskRun: { findFirst: vi.fn(async () => result) } }; } -function fakeWriter(row: { friendlyId: string } | null) { - return { taskRun: { findFirst: vi.fn(async () => row) } }; +function fakeWriter(row: { friendlyId: string; taskIdentifier?: string } | null) { + const result = row ? { ...row, taskIdentifier: row.taskIdentifier ?? "task-1" } : null; + return { taskRun: { findFirst: vi.fn(async () => result) } }; } function fakeBuffer(entry: BufferEntry | null): MollifierBuffer { @@ -47,7 +49,7 @@ describe("resolveRunForMutation", () => { getBuffer: () => null, }, }); - expect(result).toEqual({ source: "pg", friendlyId: "run_1" }); + expect(result).toEqual({ source: "pg", friendlyId: "run_1", taskIdentifier: "task-1" }); }); it("returns { source: 'buffer' } when PG misses and the buffer entry matches env+org", async () => { @@ -55,7 +57,7 @@ describe("resolveRunForMutation", () => { runId: "run_1", envId: "env_a", orgId: "org_1", - payload: "{}", + payload: JSON.stringify({ taskIdentifier: "task-1" }), status: "QUEUED", attempts: 0, createdAt: NOW, @@ -71,7 +73,11 @@ describe("resolveRunForMutation", () => { getBuffer: () => fakeBuffer(entry), }, }); - expect(result).toEqual({ source: "buffer", friendlyId: "run_1" }); + expect(result).toEqual({ + source: "buffer", + friendlyId: "run_1", + taskIdentifier: "task-1", + }); }); it("returns null when PG misses and the buffer entry env doesn't match", async () => { diff --git a/apps/webapp/test/projectEnvironmentCredentialRoute.test.ts b/apps/webapp/test/projectEnvironmentCredentialRoute.test.ts new file mode 100644 index 00000000000..4ac3cde951e --- /dev/null +++ b/apps/webapp/test/projectEnvironmentCredentialRoute.test.ts @@ -0,0 +1,127 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + authenticateEnvironmentScopedApiRequest: vi.fn<(...args: any[]) => Promise<any>>(), + authorizePatEnvironmentAccess: vi.fn<(...args: any[]) => Promise<any>>(), + authenticatedEnvironmentForAuthentication: vi.fn<(...args: any[]) => Promise<any>>(), +})); + +vi.mock("~/services/environmentVariableApiAccess.server", () => ({ + authenticateEnvironmentScopedApiRequest: mocks.authenticateEnvironmentScopedApiRequest, + authorizePatEnvironmentAccess: mocks.authorizePatEnvironmentAccess, + presentedApiKeyFromAuthentication: (authentication: any) => + authentication.type === "apiKey" && authentication.result.ok + ? authentication.result.apiKey + : undefined, +})); +vi.mock("~/services/apiAuth.server", () => ({ + authenticatedEnvironmentForAuthentication: mocks.authenticatedEnvironmentForAuthentication, + branchNameFromRequest: vi.fn(() => undefined), +})); +vi.mock("~/env.server", () => ({ + env: { API_ORIGIN: "https://api.example.com", APP_ORIGIN: "https://app.example.com" }, +})); +vi.mock("~/services/logger.server", () => ({ + logger: { error: vi.fn() }, +})); + +import { loader } from "~/routes/api.v1.projects.$projectRef.$env"; + +const environment = { + id: "env_123", + apiKey: "tr_prod_root_secret", + type: "PRODUCTION", + organizationId: "org_123", + parentEnvironment: null, + parentEnvironmentId: null, + project: { + id: "proj_123", + name: "Example project", + }, +}; + +function load() { + return loader({ + request: new Request("https://app.example.com/api/v1/projects/proj_ref/prod"), + params: { projectRef: "proj_ref", env: "prod" }, + context: {}, + }); +} + +async function responseJson(response: Response) { + return response.json() as Promise<Record<string, unknown>>; +} + +describe("project environment credential response", () => { + beforeEach(() => { + mocks.authenticateEnvironmentScopedApiRequest.mockReset(); + mocks.authorizePatEnvironmentAccess.mockReset(); + mocks.authenticatedEnvironmentForAuthentication.mockReset(); + + mocks.authorizePatEnvironmentAccess.mockResolvedValue(undefined); + mocks.authenticatedEnvironmentForAuthentication.mockResolvedValue(environment); + }); + + it("returns the presented API key", async () => { + mocks.authenticateEnvironmentScopedApiRequest.mockResolvedValue({ + ok: true, + authentication: { + type: "apiKey", + result: { + ok: true, + apiKey: "tr_prod_sk_presented", + type: "PRIVATE", + environment, + }, + }, + }); + + const response = await load(); + + expect(response.status).toBe(200); + await expect(responseJson(response)).resolves.toMatchObject({ + apiKey: "tr_prod_sk_presented", + projectId: "proj_123", + }); + }); + + it("does not exchange a grace-window root key for the current root key", async () => { + mocks.authenticateEnvironmentScopedApiRequest.mockResolvedValue({ + ok: true, + authentication: { + type: "apiKey", + result: { + ok: true, + apiKey: "tr_prod_rotated_grace_key", + type: "PRIVATE", + environment, + }, + }, + }); + + const response = await load(); + + expect(response.status).toBe(200); + await expect(responseJson(response)).resolves.toMatchObject({ + apiKey: "tr_prod_rotated_grace_key", + projectId: "proj_123", + }); + }); + + it("returns the root key to an authorized user token", async () => { + mocks.authenticateEnvironmentScopedApiRequest.mockResolvedValue({ + ok: true, + authentication: { + type: "personalAccessToken", + result: { userId: "user_123" }, + }, + }); + + const response = await load(); + + expect(response.status).toBe(200); + await expect(responseJson(response)).resolves.toMatchObject({ + apiKey: "tr_prod_root_secret", + }); + }); +}); diff --git a/apps/webapp/test/publicAccessTokenResponse.test.ts b/apps/webapp/test/publicAccessTokenResponse.test.ts new file mode 100644 index 00000000000..403167cfe1a --- /dev/null +++ b/apps/webapp/test/publicAccessTokenResponse.test.ts @@ -0,0 +1,49 @@ +import { validateJWT } from "@trigger.dev/core/v3/jwt"; +import { describe, expect, it } from "vitest"; +import { publicAccessTokenResponseHeaders } from "~/services/publicAccessTokenResponse.server"; + +describe("publicAccessTokenResponseHeaders", () => { + it("returns a server-signed token with the requested resource scopes", async () => { + const headers = await publicAccessTokenResponseHeaders({ + environment: { + id: "env_123", + apiKey: "tr_prod_root_signing_key", + }, + scopes: ["read:batch:batch_123"], + expirationTime: "1h", + }); + + expect(JSON.parse(headers["x-trigger-jwt-claims"]!)).toEqual({ + sub: "env_123", + pub: true, + }); + + const validation = await validateJWT(headers["x-trigger-jwt"]!, "tr_prod_root_signing_key"); + expect(validation.ok).toBe(true); + if (!validation.ok) return; + expect(validation.payload).toMatchObject({ + sub: "env_123", + pub: true, + scopes: ["read:batch:batch_123"], + }); + }); + + it("uses the parent signing key for branch environments", async () => { + const headers = await publicAccessTokenResponseHeaders({ + environment: { + id: "env_branch", + apiKey: "tr_preview_child_key", + parentEnvironment: { apiKey: "tr_preview_parent_key" }, + }, + scopes: ["write:waitpoints:waitpoint_123"], + expirationTime: "24h", + }); + + await expect( + validateJWT(headers["x-trigger-jwt"]!, "tr_preview_parent_key") + ).resolves.toMatchObject({ ok: true }); + await expect( + validateJWT(headers["x-trigger-jwt"]!, "tr_preview_child_key") + ).resolves.toMatchObject({ ok: false }); + }); +}); diff --git a/apps/webapp/test/rbacFallbackBranch.test.ts b/apps/webapp/test/rbacFallbackBranch.test.ts index f8e90e444ce..6745d8b117f 100644 --- a/apps/webapp/test/rbacFallbackBranch.test.ts +++ b/apps/webapp/test/rbacFallbackBranch.test.ts @@ -1,7 +1,10 @@ import { postgresTest } from "@internal/testcontainers"; import plugin from "@trigger.dev/rbac"; +import { createHash } from "node:crypto"; +import { generateJWT } from "@trigger.dev/core/v3/jwt"; import { type PrismaClient } from "@trigger.dev/database"; -import { describe, expect, vi } from "vitest"; +import { describe, expect, it, vi } from "vitest"; +import { generateAdditionalApiKey } from "~/utils/apiKeys"; import { createTestOrgProjectWithMember, uniqueId } from "./fixtures/environmentVariablesFixtures"; vi.setConfig({ testTimeout: 60_000 }); @@ -11,8 +14,11 @@ vi.setConfig({ testTimeout: 60_000 }); // mirrors findEnvironmentByApiKey, but is a separate implementation, so it // needs its own coverage. forceFallback skips loading the closed-source plugin // and uses the in-repo fallback directly. -function makeController(prisma: PrismaClient) { - return plugin.create({ primary: prisma, replica: prisma }, { forceFallback: true }); +function makeController(prisma: PrismaClient, additionalApiKeyLookupEnabled?: () => boolean) { + return plugin.create( + { primary: prisma, replica: prisma }, + { forceFallback: true, additionalApiKeyLookupEnabled } + ); } function bearerRequest(apiKey: string, branch?: string) { @@ -56,6 +62,26 @@ async function createEnv( }); } +describe("RBAC fallback — root key resolution", () => { + it("reports an unknown root key as not found", async () => { + const prisma = { + runtimeEnvironment: { findFirst: vi.fn().mockResolvedValue(null) }, + revokedApiKey: { findFirst: vi.fn().mockResolvedValue(null) }, + } as unknown as PrismaClient; + const rbac = makeController(prisma); + + const result = await rbac.authenticateBearer(bearerRequest("tr_prod_unknown")); + + expect(result).toMatchObject({ + ok: false, + resolution: { + credentialKind: "root_api_key", + lookupPath: "not_found", + }, + }); + }); +}); + describe("RBAC fallback — DEVELOPMENT branch pivot", () => { postgresTest("pivots to the named branch, carrying the parent's api key", async ({ prisma }) => { const { organization, project, orgMember } = await createTestOrgProjectWithMember(prisma); @@ -139,11 +165,392 @@ describe("RBAC fallback — DEVELOPMENT branch pivot", () => { expect(result.ok).toBe(false); if (result.ok) return; expect(result.status).toBe(401); + expect(result.error).toBe("No matching branch env"); + } + ); +}); + +describe("RBAC fallback — additional keys", () => { + it("rejects a disabled additional-key lookup without querying", async () => { + const runtimeEnvironmentFind = vi.fn(); + const revokedApiKeyFind = vi.fn(); + const apiKeyFind = vi.fn(); + const prisma = { + runtimeEnvironment: { findFirst: runtimeEnvironmentFind }, + revokedApiKey: { findFirst: revokedApiKeyFind }, + apiKey: { findFirst: apiKeyFind }, + } as unknown as PrismaClient; + const rbac = makeController(prisma, () => false); + const key = "tr_prod_sk_0123456789abcdefghijklmn"; + + await expect(rbac.authenticateBearer(bearerRequest(key))).resolves.toMatchObject({ + ok: false, + resolution: { + credentialKind: "additional_api_key", + lookupPath: "additional_skipped", + }, + }); + expect(runtimeEnvironmentFind).not.toHaveBeenCalled(); + expect(revokedApiKeyFind).not.toHaveBeenCalled(); + expect(apiKeyFind).not.toHaveBeenCalled(); + }); + + postgresTest("authenticates an additional key and records its use", async ({ prisma }) => { + const { organization, project, orgMember, user } = await createTestOrgProjectWithMember(prisma); + const rbac = makeController(prisma); + const environment = await createEnv(prisma, project.id, organization.id, { + type: "PRODUCTION", + orgMemberId: orgMember.id, + }); + const additional = generateAdditionalApiKey("PRODUCTION").apiKey; + + await prisma.apiKey.create({ + data: { + name: "External integration", + keyHash: createHash("sha256").update(additional).digest("hex"), + lastFour: additional.slice(-4), + runtimeEnvironmentId: environment.id, + createdByUserId: user.id, + presetId: null, + scopes: ["admin"], + }, + }); + const rootResult = await rbac.authenticateBearer(bearerRequest(environment.apiKey)); + const additionalResult = await rbac.authenticateBearer(bearerRequest(additional)); + + expect(rootResult.ok).toBe(true); + expect(additionalResult.ok).toBe(true); + if (!additionalResult.ok) return; + expect(additionalResult.environment.id).toBe(environment.id); + expect(additionalResult.environment.apiKey).toBe(environment.apiKey); + await expect( + prisma.apiKey.findFirst({ + where: { keyHash: createHash("sha256").update(additional).digest("hex") }, + select: { lastUsedAt: true }, + }) + ).resolves.toMatchObject({ lastUsedAt: expect.any(Date) }); + }); + + postgresTest("enforces restricted stored scopes on an additional key", async ({ prisma }) => { + const { organization, project, orgMember, user } = await createTestOrgProjectWithMember(prisma); + const rbac = makeController(prisma); + const environment = await createEnv(prisma, project.id, organization.id, { + type: "PRODUCTION", + orgMemberId: orgMember.id, + }); + const additional = generateAdditionalApiKey("PRODUCTION").apiKey; + + await prisma.apiKey.create({ + data: { + name: "Task-scoped", + keyHash: createHash("sha256").update(additional).digest("hex"), + lastFour: additional.slice(-4), + runtimeEnvironmentId: environment.id, + createdByUserId: user.id, + presetId: "TRIGGER_ONLY", + scopes: ["trigger:tasks:send-email"], + }, + }); + + const result = await rbac.authenticateBearer(bearerRequest(additional)); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.subject).toMatchObject({ type: "apiKey", restricted: true }); + expect(result.ability.can("trigger", { type: "tasks", id: "send-email" })).toBe(true); + expect(result.ability.can("trigger", { type: "tasks", id: "other-task" })).toBe(false); + expect(result.ability.can("read", { type: "runs" })).toBe(false); + }); + + postgresTest("pivots an additional key to its branch environment", async ({ prisma }) => { + const { organization, project, orgMember, user } = await createTestOrgProjectWithMember(prisma); + const rbac = makeController(prisma); + const devRoot = await createEnv(prisma, project.id, organization.id, { + type: "DEVELOPMENT", + orgMemberId: orgMember.id, + }); + const branch = await createEnv(prisma, project.id, organization.id, { + type: "DEVELOPMENT", + orgMemberId: orgMember.id, + parentEnvironmentId: devRoot.id, + branchName: "api-key-policy", + }); + const additional = generateAdditionalApiKey("DEVELOPMENT").apiKey; + + await prisma.apiKey.create({ + data: { + name: "Branch key", + keyHash: createHash("sha256").update(additional).digest("hex"), + lastFour: additional.slice(-4), + runtimeEnvironmentId: devRoot.id, + createdByUserId: user.id, + presetId: null, + scopes: ["admin"], + }, + }); + + const result = await rbac.authenticateBearer(bearerRequest(additional, "api-key-policy")); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.environment.id).toBe(branch.id); + expect(result.environment.parentEnvironment?.id).toBe(devRoot.id); + expect(result.subject).toMatchObject({ type: "apiKey", restricted: false }); + }); + + postgresTest("does not record use when branch resolution fails", async ({ prisma }) => { + const { organization, project, orgMember, user } = await createTestOrgProjectWithMember(prisma); + const rbac = makeController(prisma); + const devRoot = await createEnv(prisma, project.id, organization.id, { + type: "DEVELOPMENT", + orgMemberId: orgMember.id, + }); + const additional = generateAdditionalApiKey("DEVELOPMENT").apiKey; + const created = await prisma.apiKey.create({ + data: { + name: "Branch key", + keyHash: createHash("sha256").update(additional).digest("hex"), + lastFour: additional.slice(-4), + runtimeEnvironmentId: devRoot.id, + createdByUserId: user.id, + presetId: null, + scopes: ["admin"], + }, + }); + + const result = await rbac.authenticateBearer(bearerRequest(additional, "missing-branch")); + + expect(result).toMatchObject({ ok: false, status: 401, error: "No matching branch env" }); + await expect( + prisma.apiKey.findUnique({ where: { id: created.id }, select: { lastUsedAt: true } }) + ).resolves.toEqual({ lastUsedAt: null }); + }); + + postgresTest("treats empty stored scopes as restricted and deny-all", async ({ prisma }) => { + const { organization, project, orgMember, user } = await createTestOrgProjectWithMember(prisma); + const rbac = makeController(prisma); + const environment = await createEnv(prisma, project.id, organization.id, { + type: "PRODUCTION", + orgMemberId: orgMember.id, + }); + const additional = generateAdditionalApiKey("PRODUCTION").apiKey; + + await prisma.apiKey.create({ + data: { + name: "Empty policy", + keyHash: createHash("sha256").update(additional).digest("hex"), + lastFour: additional.slice(-4), + runtimeEnvironmentId: environment.id, + createdByUserId: user.id, + presetId: null, + scopes: [], + }, + }); + + const result = await rbac.authenticateBearer(bearerRequest(additional)); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.subject).toMatchObject({ type: "apiKey", restricted: true }); + expect(result.ability.can("read", { type: "runs" })).toBe(false); + expect(result.ability.can("trigger", { type: "tasks", id: "send-email" })).toBe(false); + }); +}); + +describe("RBAC fallback — public JWTs", () => { + postgresTest( + "keeps tokens signed with a rotated root key valid for the grace window", + async ({ prisma }) => { + const { organization, project, orgMember } = await createTestOrgProjectWithMember(prisma); + const rbac = makeController(prisma); + const environment = await createEnv(prisma, project.id, organization.id, { + type: "PRODUCTION", + orgMemberId: orgMember.id, + }); + const token = await generateJWT({ + secretKey: environment.apiKey, + payload: { pub: true, sub: environment.id, scopes: ["read:runs"] }, + expirationTime: "1h", + }); + + await expect( + rbac.authenticateBearer(bearerRequest(token), { allowJWT: true }) + ).resolves.toMatchObject({ ok: true }); + + // Rotate exactly as `regenerateApiKey` does: new value on the env, old + // value parked in RevokedApiKey with a future expiry. + const previousApiKey = environment.apiKey; + await prisma.$transaction([ + prisma.revokedApiKey.create({ + data: { + apiKey: previousApiKey, + runtimeEnvironmentId: environment.id, + expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000), + }, + }), + prisma.runtimeEnvironment.update({ + where: { id: environment.id }, + data: { apiKey: uniqueId("tr_rotated") }, + }), + ]); + + const graceResult = await rbac.authenticateBearer(bearerRequest(token), { allowJWT: true }); + expect(graceResult.ok).toBe(true); + if (!graceResult.ok) return; + expect(graceResult.environment.id).toBe(environment.id); + expect(graceResult.ability.can("read", { type: "runs" })).toBe(true); + expect(graceResult.ability.can("write", { type: "runs" })).toBe(false); + } + ); + + postgresTest( + "rejects a token signed with a rotated key once the grace window expires", + async ({ prisma }) => { + const { organization, project, orgMember } = await createTestOrgProjectWithMember(prisma); + const rbac = makeController(prisma); + const environment = await createEnv(prisma, project.id, organization.id, { + type: "PRODUCTION", + orgMemberId: orgMember.id, + }); + const token = await generateJWT({ + secretKey: environment.apiKey, + payload: { pub: true, sub: environment.id, scopes: ["read:runs"] }, + expirationTime: "1h", + }); + + await prisma.$transaction([ + prisma.revokedApiKey.create({ + data: { + apiKey: environment.apiKey, + runtimeEnvironmentId: environment.id, + expiresAt: new Date(Date.now() - 60 * 1000), + }, + }), + prisma.runtimeEnvironment.update({ + where: { id: environment.id }, + data: { apiKey: uniqueId("tr_rotated") }, + }), + ]); + + await expect( + rbac.authenticateBearer(bearerRequest(token), { allowJWT: true }) + ).resolves.toMatchObject({ ok: false, status: 401 }); + } + ); + + postgresTest("surfaces public JWT actor attribution", async ({ prisma }) => { + const { organization, project, orgMember, user } = await createTestOrgProjectWithMember(prisma); + const rbac = makeController(prisma); + const environment = await createEnv(prisma, project.id, organization.id, { + type: "PRODUCTION", + orgMemberId: orgMember.id, + }); + const token = await generateJWT({ + secretKey: environment.apiKey, + payload: { + pub: true, + sub: environment.id, + scopes: ["read:runs"], + act: { sub: user.id }, + }, + expirationTime: "1h", + }); + + const result = await rbac.authenticateBearer(bearerRequest(token), { allowJWT: true }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.jwt?.act).toEqual({ sub: user.id }); + }); + + postgresTest("rejects public JWTs for soft-deleted projects", async ({ prisma }) => { + const { organization, project, orgMember } = await createTestOrgProjectWithMember(prisma); + const rbac = makeController(prisma); + const environment = await createEnv(prisma, project.id, organization.id, { + type: "PRODUCTION", + orgMemberId: orgMember.id, + }); + const token = await generateJWT({ + secretKey: environment.apiKey, + payload: { pub: true, sub: environment.id, scopes: ["read:runs"] }, + expirationTime: "1h", + }); + await prisma.project.update({ where: { id: project.id }, data: { deletedAt: new Date() } }); + + const result = await rbac.authenticateBearer(bearerRequest(token), { allowJWT: true }); + + expect(result).toMatchObject({ ok: false, status: 401 }); + }); +}); + +describe("RBAC fallback — additional key permissions", () => { + postgresTest( + "gives additional keys root-key-equivalent permissive access", + async ({ prisma }) => { + const { organization, project, orgMember, user } = + await createTestOrgProjectWithMember(prisma); + const rbac = makeController(prisma); + const environment = await createEnv(prisma, project.id, organization.id, { + type: "PRODUCTION", + orgMemberId: orgMember.id, + }); + const additional = generateAdditionalApiKey("PRODUCTION").apiKey; + + await prisma.apiKey.create({ + data: { + name: "External integration", + keyHash: createHash("sha256").update(additional).digest("hex"), + lastFour: additional.slice(-4), + runtimeEnvironmentId: environment.id, + createdByUserId: user.id, + presetId: null, + scopes: ["admin"], + }, + }); + + const rootResult = await rbac.authenticateBearer(bearerRequest(environment.apiKey)); + const additionalResult = await rbac.authenticateBearer(bearerRequest(additional)); + + expect(rootResult.ok).toBe(true); + expect(additionalResult.ok).toBe(true); + if (!rootResult.ok || !additionalResult.ok) return; + + expect(additionalResult.subject).toMatchObject({ type: "apiKey", restricted: false }); + for (const [action, resource] of [ + ["write", { type: "envvars" }], + ["trigger", { type: "tasks", id: "send-email" }], + ["read", { type: "runs", id: "run_123" }], + ] as const) { + expect(additionalResult.ability.can(action, resource)).toBe( + rootResult.ability.can(action, resource) + ); + } } ); }); describe("RBAC fallback — branch header guards", () => { + postgresTest("preview environments require a branch header", async ({ prisma }) => { + const { organization, project } = await createTestOrgProjectWithMember(prisma); + const rbac = makeController(prisma); + const previewParent = await createEnv(prisma, project.id, organization.id, { + type: "PREVIEW", + isBranchableEnvironment: true, + }); + + const result = await rbac.authenticateBearer(bearerRequest(previewParent.apiKey)); + + expect(result).toMatchObject({ + ok: false, + status: 401, + error: "x-trigger-branch header required for preview env", + resolution: { + credentialKind: "root_api_key", + lookupPath: "root_current", + }, + }); + }); + // The "default" sentinel is DEVELOPMENT-only: it maps the dev root env to its // (branchless) self. For PREVIEW, "default" is an ordinary branch name, so a // PREVIEW branch literally named "default" is reachable and the request pivots diff --git a/apps/webapp/test/streamBatchItemsAuthorization.test.ts b/apps/webapp/test/streamBatchItemsAuthorization.test.ts new file mode 100644 index 00000000000..0ea800a1049 --- /dev/null +++ b/apps/webapp/test/streamBatchItemsAuthorization.test.ts @@ -0,0 +1,136 @@ +import { buildJwtAbility } from "@trigger.dev/plugins"; +import { describe, expect, it } from "vitest"; +import { withActionAliases } from "@trigger.dev/rbac"; +import { + authorizeBatchItems, + authorizedBatchItemStream, + batchPublicAccessScopes, +} from "~/utils/batchItemAuthorization"; +import { canWriteResolvedParentRun } from "~/utils/parentRunAuthorization"; + +async function collect(items: AsyncIterable<unknown>): Promise<unknown[]> { + const result: unknown[] = []; + for await (const item of items) result.push(item); + return result; +} + +async function* batchItems(...tasks: string[]): AsyncIterable<unknown> { + for (const task of tasks) yield { task, payload: {} }; +} + +describe("streaming batch item authorization", () => { + it("rejects a selected-task key when any streamed task is unauthorized", async () => { + const ability = withActionAliases(buildJwtAbility(["batchTrigger:tasks:task-a"])); + + await expect( + collect(authorizeBatchItems(batchItems("task-a", "task-b"), ability, "batch_123")) + ).rejects.toThrow(); + }); + + it("allows all items covered by the key's task grants", async () => { + const ability = withActionAliases( + buildJwtAbility(["batchTrigger:tasks:task-a", "batchTrigger:tasks:task-b"]) + ); + + await expect( + collect(authorizeBatchItems(batchItems("task-a", "task-b"), ability, "batch_123")) + ).resolves.toHaveLength(2); + }); + + it("preserves a public JWT's write grant for the batch", async () => { + const ability = withActionAliases(buildJwtAbility(["write:batch:batch_123"])); + + await expect( + collect(authorizeBatchItems(batchItems("task-a", "task-b"), ability, "batch_123")) + ).resolves.toHaveLength(2); + }); + + it("does not delegate batch-wide writes from selected-task credentials", () => { + const ability = withActionAliases(buildJwtAbility(["batchTrigger:tasks:task-a"])); + + expect(batchPublicAccessScopes("batch_123", ability, true)).toEqual(["read:batch:batch_123"]); + }); + + it("delegates batch-wide writes when the credential can trigger every task", () => { + const ability = withActionAliases(buildJwtAbility(["batchTrigger:tasks"])); + + expect(batchPublicAccessScopes("batch_123", ability, true)).toEqual([ + "read:batch:batch_123", + "write:batch:batch_123", + ]); + }); + + it("rejects an empty stream from a credential with no batch-level write grant", async () => { + const ability = withActionAliases(buildJwtAbility(["batchTrigger:tasks:task-a"])); + + // An empty stream authorizes nothing, so it must not reach the service — + // whose batch lookup would otherwise report the batch's existence and status. + await expect(authorizedBatchItemStream(batchItems(), ability, "batch_123")).rejects.toThrow(); + }); + + it("allows an empty stream from a credential that can write the batch", async () => { + const ability = withActionAliases(buildJwtAbility(["write:batch:batch_123"])); + + const stream = await authorizedBatchItemStream(batchItems(), ability, "batch_123"); + + await expect(collect(stream)).resolves.toHaveLength(0); + }); + + it("still authorizes every item when the first one passes", async () => { + const ability = withActionAliases(buildJwtAbility(["batchTrigger:tasks:task-a"])); + + const stream = await authorizedBatchItemStream( + batchItems("task-a", "task-b"), + ability, + "batch_123" + ); + + await expect(collect(stream)).rejects.toThrow(); + }); + + it("replays the eagerly-pulled first item to the consumer", async () => { + const ability = withActionAliases(buildJwtAbility(["batchTrigger:tasks:task-a"])); + + const stream = await authorizedBatchItemStream( + batchItems("task-a", "task-a"), + ability, + "batch_123" + ); + + await expect(collect(stream)).resolves.toEqual([ + { task: "task-a", payload: {} }, + { task: "task-a", payload: {} }, + ]); + }); + + it("treats a type-level write:runs grant as covering every parent run", () => { + // The route-level `canWriteParentRun` short-circuits on this, skipping a DB + // lookup per distinct parent. Assert the premise holds: the grant really + // does authorize an arbitrary run id. + const ability = withActionAliases(buildJwtAbility(["write:runs"])); + + expect( + canWriteResolvedParentRun(ability, { + friendlyId: "run_anything", + taskIdentifier: "some-task-we-have-no-grant-for", + }) + ).toBe(true); + }); + + it("only allows selected-task operators to link parent runs for their tasks", () => { + const ability = withActionAliases(buildJwtAbility(["write:tasks:task-a"])); + + expect( + canWriteResolvedParentRun(ability, { + friendlyId: "run_a", + taskIdentifier: "task-a", + }) + ).toBe(true); + expect( + canWriteResolvedParentRun(ability, { + friendlyId: "run_b", + taskIdentifier: "task-b", + }) + ).toBe(false); + }); +}); diff --git a/internal-packages/rbac/src/apiKeyPolicies.test.ts b/internal-packages/rbac/src/apiKeyPolicies.test.ts new file mode 100644 index 00000000000..015ba1252e1 --- /dev/null +++ b/internal-packages/rbac/src/apiKeyPolicies.test.ts @@ -0,0 +1,325 @@ +import { extractJWTSub, isPublicJWT } from "@trigger.dev/core/v3/jwt"; +import type { PrismaClient } from "@trigger.dev/database"; +import { + scopesGrantFullAccess, + type BearerAuthResult, + type RoleBaseAccessController, +} from "@trigger.dev/plugins"; +import { describe, expect, it, vi } from "vitest"; +import { buildJwtAbility } from "./ability.js"; +import loader, { resolveJwtSigningKey } from "./index.js"; + +type AuthSuccess = Extract<BearerAuthResult, { ok: true }>; + +type LazyControllerInternals = { + _init: Promise<RoleBaseAccessController>; + _hostCredentialResolver: { + authenticate: RoleBaseAccessController["authenticateBearer"]; + }; +}; + +const prismaPlaceholder = {} as PrismaClient; +const ADDITIONAL_API_KEY = "tr_prod_sk_0123456789abcdefghijklmn"; +const ROOT_API_KEY = "tr_prod_0123456789abcdefghijklmn"; +const environment = { + id: "env_123", + organizationId: "org_123", + projectId: "proj_123", + parentEnvironment: null, + parentEnvironmentId: null, +} as unknown as AuthSuccess["environment"]; + +function installPlugin( + plugin: RoleBaseAccessController, + hostAuthenticate: RoleBaseAccessController["authenticateBearer"] +) { + const controller = loader.create(prismaPlaceholder, { forceFallback: true }); + const internals = controller as unknown as LazyControllerInternals; + internals._init = Promise.resolve(plugin); + internals._hostCredentialResolver = { authenticate: hostAuthenticate }; + return controller; +} + +function additionalKeyResult(scopes: string[]): AuthSuccess { + return { + ok: true, + environment, + subject: { + type: "apiKey", + apiKeyId: "key_123", + restricted: !scopesGrantFullAccess(scopes), + organizationId: environment.organizationId, + projectId: environment.projectId, + }, + ability: buildJwtAbility(scopes), + }; +} + +function rootKeyResult(): AuthSuccess { + return { + ok: true, + environment, + subject: { + type: "user", + userId: "user_123", + organizationId: environment.organizationId, + projectId: environment.projectId, + }, + ability: buildJwtAbility(["admin"]), + }; +} + +function publicJwtResult(): AuthSuccess { + return { + ok: true, + environment, + subject: { + type: "publicJWT", + environmentId: environment.id, + organizationId: environment.organizationId, + projectId: environment.projectId, + }, + ability: buildJwtAbility(["read:runs"]), + }; +} + +function publicJwt(payload: Record<string, unknown>) { + return `header.${Buffer.from(JSON.stringify(payload)).toString("base64url")}.signature`; +} + +function bearerRequest(token: string) { + return new Request("https://api.trigger.dev/test", { + headers: { Authorization: `Bearer ${token}` }, + }); +} + +describe("API-key policy controller composition", () => { + it("routes public JWTs directly to the host without calling the plugin authenticator", async () => { + const pluginAuthenticate = vi.fn(); + const hostAuthenticate = vi.fn(async () => publicJwtResult()); + const plugin = { + isUsingPlugin: vi.fn(async () => true), + authenticateBearer: pluginAuthenticate, + } as unknown as RoleBaseAccessController; + const controller = installPlugin(plugin, hostAuthenticate); + const token = publicJwt({ pub: true, sub: environment.id }); + + const result = await controller.authenticateBearer( + new Request("https://api.trigger.dev/test", { + headers: { Authorization: `Bearer ${token}` }, + }), + { allowJWT: true } + ); + + expect(result.ok).toBe(true); + expect(hostAuthenticate).toHaveBeenCalledOnce(); + expect(pluginAuthenticate).not.toHaveBeenCalled(); + }); + + it("routes valid additional keys directly to the host and preserves their scopes", async () => { + const pluginAuthenticate = vi.fn(); + const hostAuthenticate = vi.fn(async () => additionalKeyResult(["write:tasks:send-email"])); + const plugin = { + isUsingPlugin: vi.fn(async () => true), + authenticateBearer: pluginAuthenticate, + } as unknown as RoleBaseAccessController; + const controller = installPlugin(plugin, hostAuthenticate); + + const result = await controller.authenticateBearer(bearerRequest(ADDITIONAL_API_KEY)); + + expect(result.ok).toBe(true); + expect(hostAuthenticate).toHaveBeenCalledOnce(); + expect(pluginAuthenticate).not.toHaveBeenCalled(); + if (!result.ok) return; + expect(result.subject).toMatchObject({ type: "apiKey", restricted: true }); + expect(result.ability.can("trigger", { type: "tasks", id: "send-email" })).toBe(true); + expect(result.ability.can("trigger", { type: "tasks", id: "other-task" })).toBe(false); + expect(result.ability.can("read", { type: "runs" })).toBe(false); + }); + + it("returns an unknown additional key's host 401 without calling the plugin", async () => { + const hostFailure = { + ok: false as const, + status: 401 as const, + error: "Invalid API key", + }; + const pluginAuthenticate = vi.fn(); + const hostAuthenticate = vi.fn(async () => hostFailure); + const plugin = { + isUsingPlugin: vi.fn(async () => true), + authenticateBearer: pluginAuthenticate, + } as unknown as RoleBaseAccessController; + const controller = installPlugin(plugin, hostAuthenticate); + + await expect( + controller.authenticateBearer(bearerRequest(ADDITIONAL_API_KEY)) + ).resolves.toMatchObject(hostFailure); + expect(hostAuthenticate).toHaveBeenCalledOnce(); + expect(pluginAuthenticate).not.toHaveBeenCalled(); + }); + + it("routes current root keys to the plugin without calling the host", async () => { + const pluginAuthenticate = vi.fn(async () => rootKeyResult()); + const hostAuthenticate = vi.fn(); + const plugin = { + isUsingPlugin: vi.fn(async () => true), + authenticateBearer: pluginAuthenticate, + } as unknown as RoleBaseAccessController; + const controller = installPlugin(plugin, hostAuthenticate); + + await expect(controller.authenticateBearer(bearerRequest(ROOT_API_KEY))).resolves.toMatchObject( + { + ok: true, + subject: { type: "user" }, + } + ); + expect(pluginAuthenticate).toHaveBeenCalledOnce(); + expect(hostAuthenticate).not.toHaveBeenCalled(); + }); + + it.each([401, 403] as const)( + "returns a plugin %s for root-shaped keys without calling the host", + async (status) => { + const pluginFailure = { + ok: false as const, + status, + error: "Unauthorized", + }; + const pluginAuthenticate = vi.fn(async () => pluginFailure); + const hostAuthenticate = vi.fn(); + const plugin = { + isUsingPlugin: vi.fn(async () => true), + authenticateBearer: pluginAuthenticate, + } as unknown as RoleBaseAccessController; + const controller = installPlugin(plugin, hostAuthenticate); + + await expect( + controller.authenticateBearer(bearerRequest(ROOT_API_KEY)) + ).resolves.toMatchObject(pluginFailure); + expect(pluginAuthenticate).toHaveBeenCalledOnce(); + expect(hostAuthenticate).not.toHaveBeenCalled(); + } + ); + + it("fails closed when an additional-key host route resolves a non-apiKey subject", async () => { + const pluginAuthenticate = vi.fn(); + const hostAuthenticate = vi.fn(async () => rootKeyResult()); + const plugin = { + isUsingPlugin: vi.fn(async () => true), + authenticateBearer: pluginAuthenticate, + } as unknown as RoleBaseAccessController; + const controller = installPlugin(plugin, hostAuthenticate); + + await expect( + controller.authenticateBearer(bearerRequest(ADDITIONAL_API_KEY)) + ).resolves.toMatchObject({ + ok: false, + status: 401, + error: "Invalid API key", + }); + expect(pluginAuthenticate).not.toHaveBeenCalled(); + }); + + it("keeps additional-key authentication in the no-plugin fallback controller", async () => { + const fallbackAuthenticate = vi.fn(async () => additionalKeyResult(["admin"])); + const hostAuthenticate = vi.fn(); + const fallback = { + isUsingPlugin: vi.fn(async () => false), + authenticateBearer: fallbackAuthenticate, + } as unknown as RoleBaseAccessController; + const controller = installPlugin(fallback, hostAuthenticate); + + await expect( + controller.authenticateBearer(bearerRequest(ADDITIONAL_API_KEY)) + ).resolves.toMatchObject({ ok: true, subject: { type: "apiKey" } }); + expect(fallbackAuthenticate).toHaveBeenCalledOnce(); + expect(hostAuthenticate).not.toHaveBeenCalled(); + }); + + it("delegates API-key policy catalogue, preparation, and description", async () => { + const presets = vi.fn(async () => []); + const prepare = vi.fn(async () => ({ + ok: true as const, + policy: { presetId: "FULL_ACCESS", scopes: ["admin"] }, + })); + const describePolicy = vi.fn(async () => ({ taskIdentifiers: ["send-email"] })); + const plugin = { + isUsingPlugin: vi.fn(async () => true), + apiKeyPresets: presets, + prepareApiKeyPolicy: prepare, + describeApiKeyPolicy: describePolicy, + } as unknown as RoleBaseAccessController; + const controller = installPlugin(plugin, vi.fn()); + const prepareParams = { organizationId: "org_123", presetId: "FULL_ACCESS" }; + const policy = { presetId: "TASKS", scopes: ["trigger:tasks:send-email"] }; + + await controller.apiKeyPresets("org_123"); + await controller.prepareApiKeyPolicy(prepareParams); + await controller.describeApiKeyPolicy(policy); + + expect(presets).toHaveBeenCalledWith("org_123"); + expect(prepare).toHaveBeenCalledWith(prepareParams); + expect(describePolicy).toHaveBeenCalledWith(policy); + }); +}); + +describe("API-key policy fallback", () => { + it("prepares explicit standalone full access and exposes no preset catalogue", async () => { + const controller = loader.create(prismaPlaceholder, { forceFallback: true }); + + await expect(controller.apiKeyPresets("org_123")).resolves.toBeNull(); + await expect( + controller.prepareApiKeyPolicy({ organizationId: "org_123", presetId: "FULL_ACCESS" }) + ).resolves.toEqual({ + ok: true, + policy: { presetId: null, scopes: ["admin"] }, + }); + await expect( + controller.describeApiKeyPolicy({ presetId: null, scopes: ["admin"] }) + ).resolves.toEqual({}); + }); + + it("rejects restricted presets and task input without a plugin", async () => { + const controller = loader.create(prismaPlaceholder, { forceFallback: true }); + const unavailable = { ok: false, error: "API key access presets are not available" }; + + // Anything other than full access is a restricted key, which needs the plugin. + await expect( + controller.prepareApiKeyPolicy({ organizationId: "org_123", presetId: "TRIGGER_ONLY" }) + ).resolves.toEqual(unavailable); + await expect( + controller.prepareApiKeyPolicy({ + organizationId: "org_123", + presetId: "FULL_ACCESS", + taskIdentifiers: ["send-email"], + }) + ).resolves.toEqual(unavailable); + }); +}); + +describe("scope policy helpers", () => { + it("recognizes only exact bare admin as full access", () => { + expect(scopesGrantFullAccess(["admin"])).toBe(true); + expect(scopesGrantFullAccess(["read:runs", "admin"])).toBe(true); + expect(scopesGrantFullAccess([])).toBe(false); + expect(scopesGrantFullAccess(["admin:runs"])).toBe(false); + expect(scopesGrantFullAccess(["*:all"])).toBe(false); + }); +}); + +describe("JWT host helpers", () => { + it("recognizes public JWTs and extracts their subject", () => { + const token = publicJwt({ pub: true, sub: "env_123" }); + expect(isPublicJWT(token)).toBe(true); + expect(extractJWTSub(token)).toBe("env_123"); + expect(isPublicJWT("not-a-jwt")).toBe(false); + expect(extractJWTSub("not-a-jwt")).toBeUndefined(); + }); + + it("uses the parent key as branch JWT-signing material", () => { + expect(resolveJwtSigningKey({ apiKey: "root" })).toBe("root"); + expect(resolveJwtSigningKey({ apiKey: "child", parentEnvironment: { apiKey: "parent" } })).toBe( + "parent" + ); + }); +}); diff --git a/internal-packages/rbac/src/bearerCredentials.ts b/internal-packages/rbac/src/bearerCredentials.ts new file mode 100644 index 00000000000..6b209c9f37e --- /dev/null +++ b/internal-packages/rbac/src/bearerCredentials.ts @@ -0,0 +1,433 @@ +import { isAdditionalApiKey } from "@trigger.dev/core/v3/apiKeys"; +import { extractJWTSub, isPublicJWT, validateJWT } from "@trigger.dev/core/v3/jwt"; +import { isDefaultDevBranch, sanitizeBranchName } from "@trigger.dev/core/v3/utils/gitBranch"; +import type { Prisma, PrismaClient } from "@trigger.dev/database"; +import { + scopesGrantFullAccess, + type AuthenticatedEnvironment, + type BearerAuthResult, +} from "@trigger.dev/plugins"; +import { createHash } from "node:crypto"; +import { buildJwtAbility, permissiveAbility } from "./ability.js"; + +export type BearerCredentialClients = { + // Used for the `lastUsedAt` telemetry write; reads go to the replica. + primary: PrismaClient; + replica: PrismaClient; +}; + +export type BearerCredentialKind = + | "root_api_key" + | "additional_api_key" + | "public_jwt" + | "legacy_public_key" + | "unknown"; + +export type BearerLookupPath = + | "plugin" + | "root_current" + | "root_rotated" + | "additional" + | "additional_skipped" + | "jwt_current" + | "jwt_rotated" + | "legacy_public" + | "not_found"; + +// Telemetry-only +export type BearerResolution = { + credentialKind: BearerCredentialKind; + lookupPath: BearerLookupPath; +}; + +export type BearerCredentialResult = BearerAuthResult & { + resolution: BearerResolution; +}; + +// In the future if we remove the root env.apiKey for a dedicated JWT signing +// key, this is the only function that needs to change to resolve that +export function resolveJwtSigningKey(env: { + apiKey: string; + parentEnvironment?: { apiKey: string } | null; +}): string { + return env.parentEnvironment?.apiKey ?? env.apiKey; +} + +/** + * Resolves an incoming bearer token into a `BearerAuthResult`: a public JWT, a + * root environment API key, a grace-window rotated key, or an _sk_ additional key + * + * This is deliberately NOT part of the "no-plugin fallback". Additional API + * keys are owned by the host, not by the optional RBAC plugin, so this + * resolution is always-on and is composed by *both* the fallback controller + * and the plugin-backed controller (which delegates host-owned keys here). + * + * Public JWTs carry inline scopes, which are compiled into an ability here. + * Additional API-key credentials carry DB-persisted effective scopes, which + * are compiled into their final ability here without a plugin policy lookup. + */ +export class BearerCredentialResolver { + private readonly prisma: PrismaClient; + private readonly replica: PrismaClient; + + constructor( + clients: BearerCredentialClients, + private readonly additionalApiKeyLookupEnabled: () => boolean = () => true + ) { + this.prisma = clients.primary; + this.replica = clients.replica; + } + + async authenticate( + request: Request, + options?: { allowJWT?: boolean } + ): Promise<BearerCredentialResult> { + // Deprecated public API keys (`pk_*` minted long before public JWTs + // landed) are intentionally NOT handled here. That token format hasn't + // been issued for years; any `pk_*` bearer on an apiBuilder route returns + // 401. Public access goes through the JWT path (`isPublicJWT`) instead. + const rawToken = request.headers + .get("Authorization") + ?.replace(/^Bearer /, "") + .trim(); + if (!rawToken) { + return { + ok: false, + status: 401, + error: "Invalid or Missing API key", + resolution: { credentialKind: "unknown", lookupPath: "not_found" }, + }; + } + + if (options?.allowJWT && isPublicJWT(rawToken)) { + const envId = extractJWTSub(rawToken); + if (!envId) { + return { + ok: false, + status: 401, + error: "Invalid Public Access Token", + resolution: { credentialKind: "public_jwt", lookupPath: "not_found" }, + }; + } + + // Match the include shape of the slim AuthenticatedEnvironment so + // the bridge can use the returned env without a follow-up fetch. + const env = await this.replica.runtimeEnvironment.findFirst({ + where: { id: envId }, + include: { + project: true, + organization: true, + orgMember: { + select: { + userId: true, + user: { select: { id: true, displayName: true, name: true } }, + }, + }, + parentEnvironment: { + select: { id: true, apiKey: true }, + }, + }, + }); + if (!env || env.project.deletedAt !== null) { + return { + ok: false, + status: 401, + error: "Invalid Public Access Token", + resolution: { credentialKind: "public_jwt", lookupPath: "not_found" }, + }; + } + + let lookupPath: BearerLookupPath = "jwt_current"; + let result = await validateJWT(rawToken, resolveJwtSigningKey(env)); + + // Root-key rotation grace window, mirroring the bearer path below: a + // rotated key keeps authenticating until its `RevokedApiKey` row expires, + // so tokens *signed* with that key have to keep verifying for just as + // long. Otherwise rotating a root key silently invalidates every + // outstanding public access token in the environment. + // + // Only retried on a signature mismatch — an expired or malformed token + // fails for a reason no other signing key can fix, and this runs on every + // rejected public token. + if (!result.ok && result.code === "ERR_JWS_SIGNATURE_VERIFICATION_FAILED") { + // `resolveJwtSigningKey` signs a branch with its parent's key, so the + // grace-window rows to consult belong to whichever env owns that key. + const signingEnvironmentId = env.parentEnvironment?.id ?? env.id; + const revoked = await this.replica.revokedApiKey.findMany({ + where: { + runtimeEnvironmentId: signingEnvironmentId, + expiresAt: { gt: new Date() }, + }, + select: { apiKey: true }, + }); + + if (revoked.length > 0) lookupPath = "jwt_rotated"; + + for (const candidate of revoked) { + const retried = await validateJWT(rawToken, candidate.apiKey); + if (retried.ok) { + result = retried; + break; + } + } + } + + if (!result.ok) { + return { + ok: false, + status: 401, + error: "Public Access Token is invalid", + resolution: { credentialKind: "public_jwt", lookupPath }, + }; + } + + const scopes = Array.isArray(result.payload.scopes) + ? (result.payload.scopes as string[]) + : []; + const realtime = result.payload.realtime as { skipColumns?: string[] } | undefined; + const oneTimeUse = result.payload.otu === true; + // A JWT minted from a PAT/UAT exchange stamps `act: { sub: userId }` for + // attribution. Surface it so write handlers can record the acting user. + const act = result.payload.act as { sub?: unknown } | undefined; + const actSub = typeof act?.sub === "string" ? act.sub : undefined; + + return { + ok: true, + environment: toAuthenticatedEnvironment(env), + subject: { + type: "publicJWT", + environmentId: env.id, + organizationId: env.organizationId, + projectId: env.projectId, + }, + ability: buildJwtAbility(scopes), + jwt: { realtime, oneTimeUse, ...(actSub ? { act: { sub: actSub } } : {}) }, + resolution: { credentialKind: "public_jwt", lookupPath }, + }; + } + + const branchName = sanitizeBranchName(request.headers.get("x-trigger-branch")); + + if (isAdditionalApiKey(rawToken)) { + if (!this.additionalApiKeyLookupEnabled()) { + return { + ok: false, + status: 401, + error: "Invalid API key", + resolution: { + credentialKind: "additional_api_key", + lookupPath: "additional_skipped", + }, + }; + } + + return this.resolveAdditionalKey(rawToken, branchName); + } + + return this.resolveRootKey(rawToken, branchName); + } + + private async resolveRootKey( + rawToken: string, + branchName: string | null + ): Promise<BearerCredentialResult> { + const include = environmentInclude(branchName); + const now = new Date(); + let env = await this.replica.runtimeEnvironment.findFirst({ + where: { apiKey: rawToken }, + include, + }); + + let lookupPath: BearerLookupPath = "root_current"; + + // Recently rotated root keys keep working until their grace period expires. + if (!env) { + const revoked = await this.replica.revokedApiKey.findFirst({ + where: { + apiKey: rawToken, + expiresAt: { gt: now }, + }, + include: { runtimeEnvironment: { include } }, + }); + env = revoked?.runtimeEnvironment ?? null; + lookupPath = env ? "root_rotated" : "not_found"; + } + + const resolution: BearerResolution = { + credentialKind: "root_api_key", + lookupPath, + }; + + if (!env || env.project.deletedAt !== null) { + return { + ok: false, + status: 401, + error: "Invalid API key", + resolution: { + credentialKind: "root_api_key", + lookupPath: "not_found", + }, + }; + } + + const [branchError, resolvedEnvironment] = resolveBranch(env, branchName); + if (branchError !== null) { + return { + ok: false, + status: 401, + error: branchError, + resolution, + }; + } + + return { + ok: true, + environment: toAuthenticatedEnvironment(resolvedEnvironment), + subject: { + type: "user", + userId: resolvedEnvironment.orgMember?.userId ?? "", + organizationId: resolvedEnvironment.organizationId, + projectId: resolvedEnvironment.projectId, + }, + ability: permissiveAbility, + resolution, + }; + } + + private async resolveAdditionalKey( + rawToken: string, + branchName: string | null + ): Promise<BearerCredentialResult> { + const resolution: BearerResolution = { + credentialKind: "additional_api_key", + lookupPath: "additional", + }; + const now = new Date(); + const match = await this.replica.apiKey.findFirst({ + where: { + keyHash: createHash("sha256").update(rawToken, "utf8").digest("hex"), + revokedAt: null, + OR: [{ expiresAt: null }, { expiresAt: { gt: now } }], + }, + select: { + id: true, + lastUsedAt: true, + scopes: true, + runtimeEnvironment: { include: environmentInclude(branchName) }, + }, + }); + + if (!match || match.runtimeEnvironment.project.deletedAt !== null) { + return { ok: false, status: 401, error: "Invalid API key", resolution }; + } + + const [branchError, resolvedEnvironment] = resolveBranch(match.runtimeEnvironment, branchName); + if (branchError !== null) { + return { + ok: false, + status: 401, + error: branchError, + resolution, + }; + } + + if (!match.lastUsedAt || match.lastUsedAt < new Date(now.getTime() - 300_000)) { + try { + await this.prisma.apiKey.updateMany({ + where: { + id: match.id, + revokedAt: null, + OR: [{ expiresAt: null }, { expiresAt: { gt: now } }], + }, + data: { lastUsedAt: now }, + }); + } catch { + // Authentication should not fail because last-used telemetry could not be recorded. + } + } + + return { + ok: true, + environment: toAuthenticatedEnvironment(resolvedEnvironment), + subject: { + type: "apiKey", + apiKeyId: match.id, + restricted: !scopesGrantFullAccess(match.scopes), + organizationId: resolvedEnvironment.organizationId, + projectId: resolvedEnvironment.projectId, + }, + ability: buildJwtAbility(match.scopes), + resolution, + }; + } +} + +function environmentInclude(branchName: string | null) { + return { + project: true, + organization: true, + orgMember: { + select: { + userId: true, + user: { select: { id: true, displayName: true, name: true } }, + }, + }, + parentEnvironment: { select: { id: true, apiKey: true } }, + childEnvironments: branchName ? { where: { branchName, archivedAt: null } } : undefined, + } as const satisfies Prisma.RuntimeEnvironmentInclude; +} + +type EnvironmentWithBranches = Prisma.RuntimeEnvironmentGetPayload<{ + include: ReturnType<typeof environmentInclude>; +}>; + +type BranchResolution = + | [error: string, environment: null] + | [error: null, environment: AuthenticatedEnvironment]; + +function resolveBranch( + environment: EnvironmentWithBranches, + branchName: string | null +): BranchResolution { + if (environment.type === "PREVIEW" && !branchName) { + return ["x-trigger-branch header required for preview env", null]; + } + + if (environment.type === "PREVIEW" || environment.type === "DEVELOPMENT") { + // The "default" root branch is DEVELOPMENT-only: it maps to the dev root env + // (which carries no branch), so we skip the pivot there. For PREVIEW, + // "default" is an ordinary branch name and must still pivot to its child. + const isDevAndDefault = environment.type === "DEVELOPMENT" && isDefaultDevBranch(branchName); + if (branchName !== null && !isDevAndDefault) { + const child = environment.childEnvironments[0]; + if (!child) { + return ["No matching branch env", null]; + } + + return [ + null, + { + ...child, + apiKey: environment.apiKey, + orgMember: environment.orgMember, + organization: environment.organization, + project: environment.project, + parentEnvironment: { id: environment.id, apiKey: environment.apiKey }, + }, + ]; + } + } + + return [null, environment]; +} + +// Coerce Prisma's Decimal value to a number at the authentication boundary. +function toAuthenticatedEnvironment( + environment: AuthenticatedEnvironment +): AuthenticatedEnvironment { + const burst = environment.concurrencyLimitBurstFactor; + return { + ...environment, + concurrencyLimitBurstFactor: typeof burst === "number" ? burst : burst.toNumber(), + }; +} diff --git a/internal-packages/rbac/src/fallback.ts b/internal-packages/rbac/src/fallback.ts index 4cf1e765c7a..9054c0a43e1 100644 --- a/internal-packages/rbac/src/fallback.ts +++ b/internal-packages/rbac/src/fallback.ts @@ -1,7 +1,6 @@ import type { Permission, Role, - RbacEnvironment, RbacUser, RbacSubject, RbacResource, @@ -20,9 +19,8 @@ import { } from "@trigger.dev/plugins"; import { createHash } from "node:crypto"; import type { PrismaClient } from "@trigger.dev/database"; -import { validateJWT } from "@trigger.dev/core/v3/jwt"; -import { isDefaultDevBranch, sanitizeBranchName } from "@trigger.dev/core/v3/utils/gitBranch"; -import { buildFallbackAbility, buildJwtAbility, permissiveAbility } from "./ability.js"; +import { buildFallbackAbility, permissiveAbility } from "./ability.js"; +import { BearerCredentialResolver } from "./bearerCredentials.js"; export type FallbackPrismaClients = { // Used for writes (setUserRole, mutateRole, etc.) and any reads that @@ -48,6 +46,7 @@ function resolvePrismaClients(input: PrismaInput): FallbackPrismaClients { export type FallbackOptions = { // Platform secret for verifying delegated user-actor tokens (tr_uat_). userActorSecret?: string; + additionalApiKeyLookupEnabled?: () => boolean; }; export class RoleBaseAccessFallback { @@ -68,11 +67,15 @@ class RoleBaseAccessFallbackController implements RoleBaseAccessController { private readonly prisma: PrismaClient; // alias for primary — used by writes private readonly replica: PrismaClient; private readonly userActorSecret?: string; + // Bearer-token resolution (JWTs, root keys, additional API keys) is always-on + // host logic, not an RBAC default — see bearerCredentials.ts. + private readonly bearer: BearerCredentialResolver; constructor(clients: FallbackPrismaClients, options?: FallbackOptions) { this.prisma = clients.primary; this.replica = clients.replica; this.userActorSecret = options?.userActorSecret; + this.bearer = new BearerCredentialResolver(clients, options?.additionalApiKeyLookupEnabled); } async isUsingPlugin(): Promise<boolean> { @@ -83,166 +86,7 @@ class RoleBaseAccessFallbackController implements RoleBaseAccessController { request: Request, options?: { allowJWT?: boolean } ): Promise<BearerAuthResult> { - // Deprecated public API keys (`pk_*` minted long before public JWTs - // landed) are intentionally NOT handled here. The legacy - // `findEnvironmentByPublicApiKey` path looked them up via the - // `pkApiKey` column, but that token format hasn't been issued for - // years and no live client should be sending one. Any `pk_*` bearer - // on a route that goes through the apiBuilder now returns 401 — - // public access goes through the JWT path (`isPublicJWT(rawToken)` - // below) instead. The deprecated lookup is still exported from - // `apps/webapp/app/models/runtimeEnvironment.server.ts` for the - // pre-RBAC routes that haven't been migrated, but it's a dead - // code path for any route that uses `createLoaderApiRoute` / - // `createActionApiRoute`. - const rawToken = request.headers - .get("Authorization") - ?.replace(/^Bearer /, "") - .trim(); - if (!rawToken) return { ok: false, status: 401, error: "Invalid or Missing API key" }; - - if (options?.allowJWT && isPublicJWT(rawToken)) { - const envId = extractJWTSub(rawToken); - if (!envId) return { ok: false, status: 401, error: "Invalid Public Access Token" }; - - // Match the include shape of the slim AuthenticatedEnvironment so - // the bridge can use the returned env without a follow-up fetch. - const env = await this.replica.runtimeEnvironment.findFirst({ - where: { id: envId }, - include: { - project: true, - organization: true, - orgMember: { - select: { - userId: true, - user: { select: { id: true, displayName: true, name: true } }, - }, - }, - parentEnvironment: { select: { id: true, apiKey: true } }, - }, - }); - if (!env || env.project.deletedAt !== null) { - return { ok: false, status: 401, error: "Invalid Public Access Token" }; - } - - const signingKey = env.parentEnvironment?.apiKey ?? env.apiKey; - const result = await validateJWT(rawToken, signingKey); - if (!result.ok) return { ok: false, status: 401, error: "Public Access Token is invalid" }; - - const scopes = Array.isArray(result.payload.scopes) - ? (result.payload.scopes as string[]) - : []; - const realtime = result.payload.realtime as { skipColumns?: string[] } | undefined; - const oneTimeUse = result.payload.otu === true; - // A JWT minted from a PAT/UAT exchange stamps `act: { sub: userId }` for - // attribution. Surface it so write handlers can record the acting user. - const act = result.payload.act as { sub?: unknown } | undefined; - const actSub = typeof act?.sub === "string" ? act.sub : undefined; - - return { - ok: true, - environment: toAuthenticatedEnvironment(env), - subject: { - type: "publicJWT", - environmentId: env.id, - organizationId: env.organizationId, - projectId: env.projectId, - }, - ability: buildJwtAbility(scopes), - jwt: { realtime, oneTimeUse, ...(actSub ? { act: { sub: actSub } } : {}) }, - }; - } - - // PREVIEW (and DEVELOPMENT) envs are parents — operating "on a branch" means routing - // to a child env keyed by branchName. The customer authenticates - // with the parent's apiKey + an `x-trigger-branch` header. Mirror - // findEnvironmentByApiKey: include the matching child env so the - // pivot below can adopt its identity. - const branchName = sanitizeBranchName(request.headers.get("x-trigger-branch")); - // Match the include shape of the slim AuthenticatedEnvironment so - // the apiBuilder bridge can use the returned env directly without a - // follow-up findEnvironmentById call. - const include = { - project: true, - organization: true, - orgMember: { - select: { - userId: true, - user: { select: { id: true, displayName: true, name: true } }, - }, - }, - parentEnvironment: { select: { id: true, apiKey: true } }, - childEnvironments: branchName ? { where: { branchName, archivedAt: null } } : undefined, - } as const; - let env = await this.replica.runtimeEnvironment.findFirst({ - where: { apiKey: rawToken }, - include, - }); - - // Revoked API key grace window — mirrors `findEnvironmentByApiKey` - // in apps/webapp/app/models/runtimeEnvironment.server.ts. Recently - // rotated keys keep working until their `expiresAt`; without this - // branch a customer who rotates an env API key gets immediate 401s - // on the new auth path. The PR's e2e suite covers this in - // auth-cross-cutting.e2e.full.test.ts ("revoked key within grace"). - if (!env) { - const revoked = await this.replica.revokedApiKey.findFirst({ - where: { apiKey: rawToken, expiresAt: { gt: new Date() } }, - include: { runtimeEnvironment: { include } }, - }); - env = revoked?.runtimeEnvironment ?? null; - } - - if (!env || env.project.deletedAt !== null) { - return { ok: false, status: 401, error: "Invalid API key" }; - } - - if (env.type === "PREVIEW" && !branchName) { - return { - ok: false, - status: 401, - error: "x-trigger-branch header required for preview env", - }; - } - - if (env.type === "PREVIEW" || env.type === "DEVELOPMENT") { - // The "default" root branch is DEVELOPMENT-only: it maps to the dev root env - // (which carries no branch), so we skip the pivot there. For PREVIEW, - // "default" is an ordinary branch name and must still pivot to its child. - const isDevAndDefault = env.type === "DEVELOPMENT" && isDefaultDevBranch(branchName); - if (branchName !== null && !isDevAndDefault) { - const child = env.childEnvironments?.[0]; - if (!child) { - return { ok: false, status: 401, error: "No matching branch env" }; - } - // Pivot to the child env: child's id/type/branchName, parent's - // apiKey/orgMember/organization/project. parentEnvironment is set - // explicitly here so the slim shape stays internally consistent. - env = { - ...child, - apiKey: env.apiKey, - orgMember: env.orgMember, - organization: env.organization, - project: env.project, - parentEnvironment: { id: env.id, apiKey: env.apiKey }, - childEnvironments: [], - }; - } - } - - const subject: RbacSubject = { - type: "user", - userId: env.orgMember?.userId ?? "", - organizationId: env.organizationId, - projectId: env.projectId, - }; - - return { - ok: true, - environment: toAuthenticatedEnvironment(env), - subject, - ability: permissiveAbility, - }; + return this.bearer.authenticate(request, options); } async authenticateSession( @@ -388,14 +232,14 @@ class RoleBaseAccessFallbackController implements RoleBaseAccessController { taskIdentifiers?: string[]; }) { // Without a plugin there is no preset catalogue, so full access is the only - // policy on offer, but the caller still has to ask for it by name. Any + // policy on offer — but the caller still has to ask for it by name. Any // other preset, or any task selection, is a restricted key and unavailable. if (params.presetId !== FULL_ACCESS_PRESET_ID || (params.taskIdentifiers?.length ?? 0) > 0) { return { ok: false as const, error: "API key access presets are not available" }; } - // `presetId: null` because this install has no catalogue to reference. The - // persisted scopes remain the source of truth for authorization. + // `presetId: null` because this install has no catalogue to reference — the + // key is full-access, not an instance of a named preset. return { ok: true as const, policy: { presetId: null, scopes: ["admin"] }, @@ -463,48 +307,6 @@ class RoleBaseAccessFallbackController implements RoleBaseAccessController { } } -function isPublicJWT(token: string): boolean { - const parts = token.split("."); - if (parts.length !== 3) return false; - try { - const payload = JSON.parse( - Buffer.from(parts[1].replace(/-/g, "+").replace(/_/g, "/"), "base64").toString("utf8") - ); - return payload !== null && typeof payload === "object" && payload.pub === true; - } catch { - return false; - } -} - -function extractJWTSub(token: string): string | undefined { - const parts = token.split("."); - if (parts.length !== 3) return undefined; - try { - const payload = JSON.parse( - Buffer.from(parts[1].replace(/-/g, "+").replace(/_/g, "/"), "base64").toString("utf8") - ); - return payload !== null && typeof payload === "object" && typeof payload.sub === "string" - ? payload.sub - : undefined; - } catch { - return undefined; - } -} - -// Coerce a Prisma RuntimeEnvironment payload (with project/organization/ -// orgMember/parentEnvironment includes) into the slim AuthenticatedEnvironment -// the auth contract carries. The slim type accepts both `number` and -// Decimal-like for `concurrencyLimitBurstFactor`, but explicit coercion -// here keeps the value a plain number across the auth boundary so -// downstream consumers don't have to narrow before doing arithmetic. -function toAuthenticatedEnvironment(env: RbacEnvironment): RbacEnvironment { - const burst = env.concurrencyLimitBurstFactor; - return { - ...env, - concurrencyLimitBurstFactor: typeof burst === "number" ? burst : burst.toNumber(), - }; -} - function toRbacUser(user: { id: string; email: string; diff --git a/internal-packages/rbac/src/index.ts b/internal-packages/rbac/src/index.ts index 6873c9d479f..bef649a6c9e 100644 --- a/internal-packages/rbac/src/index.ts +++ b/internal-packages/rbac/src/index.ts @@ -1,6 +1,7 @@ import type { ApiKeyPolicyDescription, ApiKeyPreset, + BearerAuthResult, Permission, PrepareApiKeyPolicyResult, RbacAbility, @@ -13,8 +14,17 @@ import type { RoleMutationResult, } from "@trigger.dev/plugins"; import type { PrismaClient } from "@trigger.dev/database"; +import { isAdditionalApiKey } from "@trigger.dev/core/v3/apiKeys"; +import { isPublicJWT } from "@trigger.dev/core/v3/jwt"; + import { RoleBaseAccessFallback } from "./fallback.js"; +import { BearerCredentialResolver, type BearerResolution } from "./bearerCredentials.js"; export type { RoleBaseAccessController, RbacAbility, RbacResource } from "@trigger.dev/plugins"; +export type { + BearerCredentialKind, + BearerLookupPath, + BearerResolution, +} from "./bearerCredentials.js"; /** * The controller surface as the HOST sees it, after LazyController has filled in @@ -26,10 +36,18 @@ export type { RoleBaseAccessController, RbacAbility, RbacResource } from "@trigg * method — so host consumers should depend on this type, not on the plugin * contract, and get a total surface without writing their own guards. */ -export type HostRbacController = Required<RoleBaseAccessController>; +export type HostBearerAuthResult = BearerAuthResult & { resolution: BearerResolution }; + +export type HostRbacController = Omit<Required<RoleBaseAccessController>, "authenticateBearer"> & { + authenticateBearer( + request: Request, + options?: { allowJWT?: boolean } + ): Promise<HostBearerAuthResult>; +}; export type { UserActorAuthResult, UserActorClaims } from "@trigger.dev/plugins"; export { buildJwtAbility, scopesWithinAbility } from "./ability.js"; export { FULL_ACCESS_PRESET_ID, scopesGrantFullAccess } from "@trigger.dev/plugins"; +export { resolveJwtSigningKey } from "./bearerCredentials.js"; // Re-export the user-actor token grammar so the webapp mints/checks tokens // through @trigger.dev/rbac (it doesn't import @trigger.dev/plugins directly). export { @@ -56,6 +74,9 @@ export type RbacCreateOptions = { // follows the host's writer/replica topology. The fallback ignores this — // it queries through the Prisma clients passed as `RbacPrismaInput`. database?: RbacDatabaseConfig; + // Synchronous host-owned rollout control. Defaults to enabled for non-webapp + // consumers; the webapp passes its cold-safe global flag reader. + additionalApiKeyLookupEnabled?: () => boolean; }; // Route actions that historically authorised via the legacy checkAuthorization's @@ -81,12 +102,47 @@ export function withActionAliases(underlying: RbacAbility): RbacAbility { }; } +function resolveBearerResolution( + result: BearerAuthResult, + context: { + useHostForAdditionalKey: boolean; + useHostForPublicJWT: boolean; + rawToken: string | undefined; + usingPlugin: boolean; + } +): BearerResolution { + if ("resolution" in result) { + return result.resolution as BearerResolution; + } + + if (context.useHostForAdditionalKey) { + return { credentialKind: "additional_api_key", lookupPath: "additional" }; + } + + if (context.useHostForPublicJWT) { + return { credentialKind: "public_jwt", lookupPath: "jwt_current" }; + } + + return { + credentialKind: context.rawToken?.startsWith("tr_") ? "root_api_key" : "unknown", + lookupPath: context.usingPlugin ? "plugin" : "not_found", + }; +} + // Loads the plugin lazily; falls back to the fallback implementation if not installed. // Synchronous create() avoids top-level await (not supported in the webapp's CJS build). class LazyController implements RoleBaseAccessController { private readonly _init: Promise<RoleBaseAccessController>; + // Additional API keys (the ApiKey table) are host-owned, not known to the + // optional plugin. The host resolves them with its always-on credential + // resolver — not a full RBAC fallback controller. + private readonly _hostCredentialResolver: BearerCredentialResolver; constructor(prisma: RbacPrismaInput, options?: RbacCreateOptions) { + this._hostCredentialResolver = new BearerCredentialResolver( + "primary" in prisma ? prisma : { primary: prisma, replica: prisma }, + options?.additionalApiKeyLookupEnabled + ); this._init = this.load(prisma, options); // load() runs eagerly but the result is awaited lazily on first method // call. If load() rejects (e.g. REQUIRE_PLUGINS=1 + plugin missing) and @@ -104,6 +160,7 @@ class LazyController implements RoleBaseAccessController { if (options?.forceFallback) { return new RoleBaseAccessFallback(prisma, { userActorSecret: options?.userActorSecret, + additionalApiKeyLookupEnabled: options?.additionalApiKeyLookupEnabled, }).create(); } const moduleName = "@triggerdotdev/plugins/rbac"; @@ -163,6 +220,7 @@ class LazyController implements RoleBaseAccessController { return new RoleBaseAccessFallback(prisma, { userActorSecret: options?.userActorSecret, + additionalApiKeyLookupEnabled: options?.additionalApiKeyLookupEnabled, }).create(); } } @@ -176,8 +234,50 @@ class LazyController implements RoleBaseAccessController { } async authenticateBearer(...args: Parameters<RoleBaseAccessController["authenticateBearer"]>) { - const result = await (await this.c()).authenticateBearer(...args); - return result.ok ? { ...result, ability: withActionAliases(result.ability) } : result; + const controller = await this.c(); + const usingPlugin = await controller.isUsingPlugin(); + const [request, options] = args; + const rawToken = request.headers + .get("Authorization") + ?.replace(/^Bearer /, "") + .trim(); + const useHostForPublicJWT = Boolean(options?.allowJWT && rawToken && isPublicJWT(rawToken)); + const useHostForAdditionalKey = Boolean( + !useHostForPublicJWT && usingPlugin && rawToken && isAdditionalApiKey(rawToken) + ); + + // Public JWT validation and additional environment API keys are host-owned. + // Route those formats directly to the host; all other bearer credentials + // remain authoritative in the installed controller. + const result = + useHostForPublicJWT || useHostForAdditionalKey + ? await this._hostCredentialResolver.authenticate(...args) + : await controller.authenticateBearer(...args); + + const resolution = resolveBearerResolution(result, { + useHostForAdditionalKey, + useHostForPublicJWT, + rawToken, + usingPlugin, + }); + + // The format is only a routing hint. A successful host resolution on the + // additional-key path must still produce the expected principal type. + if (useHostForAdditionalKey && result.ok && result.subject.type !== "apiKey") { + return { + ok: false as const, + status: 401 as const, + error: "Invalid API key", + resolution: { + credentialKind: "additional_api_key" as const, + lookupPath: "additional" as const, + }, + }; + } + + return result.ok + ? { ...result, ability: withActionAliases(result.ability), resolution } + : { ...result, resolution }; } async authenticateSession(...args: Parameters<RoleBaseAccessController["authenticateSession"]>) { diff --git a/packages/core/package.json b/packages/core/package.json index bd890b4e582..292d4ff421d 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -31,6 +31,7 @@ "./v3/build": "./src/v3/build/index.ts", "./v3/apps": "./src/v3/apps/index.ts", "./v3/auth/environment": "./src/v3/auth/environment.ts", + "./v3/apiKeys": "./src/v3/apiKeys.ts", "./v3/jwt": "./src/v3/jwt.ts", "./v3/errors": "./src/v3/errors.ts", "./v3/logger-api": "./src/v3/logger-api.ts", @@ -151,6 +152,9 @@ "v3/jwt": [ "dist/commonjs/v3/jwt.d.ts" ], + "v3/apiKeys": [ + "dist/commonjs/v3/apiKeys.d.ts" + ], "v3/runEngineWorker": [ "dist/commonjs/v3/runEngineWorker/index.d.ts" ], @@ -358,6 +362,17 @@ "default": "./dist/commonjs/v3/auth/environment.js" } }, + "./v3/apiKeys": { + "import": { + "@triggerdotdev/source": "./src/v3/apiKeys.ts", + "types": "./dist/esm/v3/apiKeys.d.ts", + "default": "./dist/esm/v3/apiKeys.js" + }, + "require": { + "types": "./dist/commonjs/v3/apiKeys.d.ts", + "default": "./dist/commonjs/v3/apiKeys.js" + } + }, "./v3/jwt": { "import": { "@triggerdotdev/source": "./src/v3/jwt.ts", diff --git a/packages/core/src/v3/schemas/api.ts b/packages/core/src/v3/schemas/api.ts index 3b57395b297..3f6ba0aeced 100644 --- a/packages/core/src/v3/schemas/api.ts +++ b/packages/core/src/v3/schemas/api.ts @@ -430,6 +430,8 @@ export type BatchTriggerTaskV3Response = z.infer<typeof BatchTriggerTaskV3Respon export const CreateBatchRequestBody = z.object({ /** Expected number of items in the batch */ runCount: z.number().int().positive(), + /** Distinct task identifiers expected in the item stream */ + taskIdentifiers: z.array(z.string().min(1)).min(1).optional(), /** Parent run ID for batchTriggerAndWait (friendly ID) */ parentRunId: z.string().optional(), /** Whether to resume parent on completion (true for batchTriggerAndWait) */ diff --git a/packages/core/src/v3/schemas/idempotencyKey.test.ts b/packages/core/src/v3/schemas/idempotencyKey.test.ts index cad9db69080..0ab40a4850e 100644 --- a/packages/core/src/v3/schemas/idempotencyKey.test.ts +++ b/packages/core/src/v3/schemas/idempotencyKey.test.ts @@ -97,6 +97,24 @@ describe("idempotencyKey length validation", () => { expect(result.success).toBe(true); }); + + it("accepts a non-empty task identifier declaration", () => { + const result = CreateBatchRequestBody.safeParse({ + runCount: 2, + taskIdentifiers: ["task-a", "task-b"], + }); + + expect(result.success).toBe(true); + }); + + it("rejects an empty task identifier declaration", () => { + const result = CreateBatchRequestBody.safeParse({ + runCount: 1, + taskIdentifiers: [], + }); + + expect(result.success).toBe(false); + }); }); describe("CreateWaitpointTokenRequestBody", () => { diff --git a/packages/trigger-sdk/src/v3/shared.test.ts b/packages/trigger-sdk/src/v3/shared.test.ts index 3eb7e821b04..ff5d694edf5 100644 --- a/packages/trigger-sdk/src/v3/shared.test.ts +++ b/packages/trigger-sdk/src/v3/shared.test.ts @@ -1,6 +1,22 @@ import { ApiClient } from "@trigger.dev/core/v3"; import { describe, it, expect } from "vitest"; -import { offloadBatchItemPayloads, readableStreamToAsyncIterable } from "./shared.js"; +import { + offloadBatchItemPayloads, + readableStreamToAsyncIterable, + uniqueBatchTaskIdentifiers, +} from "./shared.js"; + +describe("uniqueBatchTaskIdentifiers", () => { + it("returns a stable, deduplicated declaration for batch creation", () => { + expect( + uniqueBatchTaskIdentifiers([ + { index: 0, task: "task-b", payload: "{}" }, + { index: 1, task: "task-a", payload: "{}" }, + { index: 2, task: "task-b", payload: "{}" }, + ]) + ).toEqual(["task-a", "task-b"]); + }); +}); describe("offloadBatchItemPayloads", () => { // A real client is required for conditionallyExportPacket's truthy check; small payloads diff --git a/packages/trigger-sdk/src/v3/shared.ts b/packages/trigger-sdk/src/v3/shared.ts index 13d402be6ac..50ab312eb39 100644 --- a/packages/trigger-sdk/src/v3/shared.ts +++ b/packages/trigger-sdk/src/v3/shared.ts @@ -1631,6 +1631,10 @@ export async function batchTriggerAndWaitTasks<TTasks extends readonly AnyTask[] } } +export function uniqueBatchTaskIdentifiers(items: BatchItemNDJSON[]): string[] { + return Array.from(new Set(items.map((item) => item.task))).sort(); +} + /** * Helper function that executes a 2-phase batch trigger: * 1. Creates the batch record with expected run count @@ -1678,6 +1682,7 @@ async function executeBatchTwoPhase( batch = await apiClient.createBatch( { runCount: items.length, + taskIdentifiers: uniqueBatchTaskIdentifiers(items), parentRunId: options.parentRunId, resumeParentOnCompletion: options.resumeParentOnCompletion, idempotencyKey: options.idempotencyKey, From 859f30e224197b39f31d9080dbfce238631996c0 Mon Sep 17 00:00:00 2001 From: Katia Bulatova <katia@trigger.dev> Date: Mon, 3 Aug 2026 16:12:03 +0200 Subject: [PATCH 61/67] fix(webapp): report message catalogs survive the production bundle (#4488) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /api/v1/reports/health threw `no catalog registered for report "health"` in production (fine in dev): the catalog registered itself as a side effect of a bare import, which the SSR build tree-shakes under `"sideEffects": false`. Verified on the built server bundle — main's is missing the catalog, this branch's carries it. Fix: catalogs are values on the report registry entries; the resolver reads them from there and the mutable register-at-import step is gone. --- .server-changes/report-messages-not-treeshaken.md | 6 ++++++ .../v3/reports/health/health-messages.ts | 4 +--- .../app/presenters/v3/reports/health/health.ts | 1 - .../v3/reports/report-message-catalogs.ts | 11 +++++++++++ .../app/presenters/v3/reports/report-messages.ts | 14 +++++--------- 5 files changed, 23 insertions(+), 13 deletions(-) create mode 100644 .server-changes/report-messages-not-treeshaken.md create mode 100644 apps/webapp/app/presenters/v3/reports/report-message-catalogs.ts diff --git a/.server-changes/report-messages-not-treeshaken.md b/.server-changes/report-messages-not-treeshaken.md new file mode 100644 index 00000000000..b599e84f1ce --- /dev/null +++ b/.server-changes/report-messages-not-treeshaken.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Fix the health report failing with an internal error when requested through the API. diff --git a/apps/webapp/app/presenters/v3/reports/health/health-messages.ts b/apps/webapp/app/presenters/v3/reports/health/health-messages.ts index 888b2b54b73..2475118a685 100644 --- a/apps/webapp/app/presenters/v3/reports/health/health-messages.ts +++ b/apps/webapp/app/presenters/v3/reports/health/health-messages.ts @@ -8,7 +8,7 @@ * metrics / evidence — meaning lives here, numbers stay facts. */ -import { registerReportMessages, type ReportMessages } from "../report-messages"; +import { type ReportMessages } from "../report-messages"; import { type ReasonCode, type Severity } from "../report-view-model"; /** Metric id -> expanded display label. */ @@ -166,5 +166,3 @@ export const healthMessages: ReportMessages = { statementMessage, actionMessage: (code) => ACTIONS[code] ?? code, }; - -registerReportMessages("health", healthMessages); diff --git a/apps/webapp/app/presenters/v3/reports/health/health.ts b/apps/webapp/app/presenters/v3/reports/health/health.ts index 08beae3894c..fa56e7b630e 100644 --- a/apps/webapp/app/presenters/v3/reports/health/health.ts +++ b/apps/webapp/app/presenters/v3/reports/health/health.ts @@ -26,7 +26,6 @@ import { applyFlowPolicy, buildFlowRead, interpretFlow } from "./flow"; import { interpretLiveness } from "./liveness"; // Registers the "health" message catalog (side effect) so the renderer resolves this report's // codes. Kept here — the health report's entry module — so loading it always registers its prose. -import "./health-messages"; // Re-exported so the data layer + tests keep a single import path (`./health`). export { HEALTH_THRESHOLDS, isPendingIncreasing, type HealthInput } from "./health-core"; diff --git a/apps/webapp/app/presenters/v3/reports/report-message-catalogs.ts b/apps/webapp/app/presenters/v3/reports/report-message-catalogs.ts new file mode 100644 index 00000000000..eca36e7e95a --- /dev/null +++ b/apps/webapp/app/presenters/v3/reports/report-message-catalogs.ts @@ -0,0 +1,11 @@ +/** + * Catalogs by value, in a module that imports ONLY the per-report `*-messages` + * files — no loaders, no IO. Presentation stays decoupled from the data layer, + * and a value import can't be tree-shaken away. + */ +import { healthMessages } from "./health/health-messages"; +import { type ReportMessages } from "./report-messages"; + +export const REPORT_MESSAGE_CATALOGS: Record<string, ReportMessages> = { + health: healthMessages, +}; diff --git a/apps/webapp/app/presenters/v3/reports/report-messages.ts b/apps/webapp/app/presenters/v3/reports/report-messages.ts index 2d90ea016ed..1c549f78a1c 100644 --- a/apps/webapp/app/presenters/v3/reports/report-messages.ts +++ b/apps/webapp/app/presenters/v3/reports/report-messages.ts @@ -5,6 +5,7 @@ * No report vocabulary here — that would re-couple the renderer to a specific report. */ +import { REPORT_MESSAGE_CATALOGS } from "./report-message-catalogs"; import { type ReasonCode, type Severity } from "./report-view-model"; /** @@ -22,16 +23,11 @@ export type ReportMessages = { actionMessage(code: ReasonCode): string; }; -const catalogs = new Map<string, ReportMessages>(); - -/** Register a report's catalog under its title (e.g. "health"). Called for its side effect. */ -export function registerReportMessages(title: string, messages: ReportMessages): void { - catalogs.set(title, messages); -} - -/** Look up a report's catalog by `vm.title`. Throws if the report never registered one. */ +/** Look up a report's catalog by `vm.title`. Catalogs are values, never + * registered at import time — side-effect registration is what the production + * bundle tree-shakes away. */ export function reportMessages(title: string): ReportMessages { - const messages = catalogs.get(title); + const messages = REPORT_MESSAGE_CATALOGS[title]; if (!messages) { throw new Error(`report-messages: no catalog registered for report "${title}"`); } From 75df940e4cfbe610a6d28510aac550ae45b08ffd Mon Sep 17 00:00:00 2001 From: Matt Aitken <matt@mattaitken.com> Date: Mon, 3 Aug 2026 15:34:17 +0100 Subject: [PATCH 62/67] chore: vouch Leafgard (#4489) Adds `Leafgard` to the list of vouched outside contributors so their PRs aren't auto-closed by the vouch check. Closes #4487 --- .github/VOUCHED.td | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/VOUCHED.td b/.github/VOUCHED.td index 7237a5369ae..a2ebac99dd8 100644 --- a/.github/VOUCHED.td +++ b/.github/VOUCHED.td @@ -24,4 +24,5 @@ jrossi ThullyoCunha ConProgramming saasjesus -brentshulman-silkline \ No newline at end of file +brentshulman-silkline +Leafgard \ No newline at end of file From 9d57aff5426be111bbf3315c8167c754d4c25c10 Mon Sep 17 00:00:00 2001 From: Eric Allam <eric@trigger.dev> Date: Mon, 3 Aug 2026 16:19:50 +0100 Subject: [PATCH 63/67] fix(webapp): make the Queues hero charts environment-wide (#4486) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary The four charts above the queues table aggregated over **at most the 25 queues on the current page**. They reused the loader's already-paginated queue array as a ClickHouse `queue IN (...)` filter, so paging or re-sorting changed the values, and a name search matching nothing blanked the whole chart row. The stat tiles above them were already environment-wide, so the two rows disagreed. They now read `env_metrics`, the environment-level rollup that already exists for exactly this (the built-in Queues dashboard and the health report read it). That is both correct and queue-count-independent: no `GROUP BY queue` across an entire environment, and no client-side summing. Note this is not only a paging artifact: page 1 under-reported too. On the seeded environment below, page 1 read 82% saturation against a true 87%, because the environment's running total is not the sum of one page of per-queue gauges. Three related fixes ride along. **Scheduling delay and throttling sawed to zero.** Both are event-driven, so at the 10-second bucket a short range picks, most buckets hold no samples at all and were drawn as `0ms`. Measured over a 1-hour window: **232 of 349 buckets had no scheduling-delay samples**. A bucket where nothing started is not a bucket where nothing waited, so the line was both ugly and wrong. TRQL grows a `minBucketSeconds` floor, plumbed through the metric resource route, and the hero tiles set 60s. Buckets that still have no samples render as a gap instead of a dive to zero. **The floor must not feed a width-dependent headline.** Two of the four headlines are not peaks, so widening the plotted buckets moved them: - **Throttled** is a share of buckets that saw any throttling, so a single brief throttle came to mark a whole minute instead of ten seconds: the same seeded events read 17% at 10s and 85% at 60s. - **Scheduling delay p95** is a percentile, and merging quantile states over a wider bucket yields a p95 between the sub-buckets' own. Two 240s samples among twenty in one 10-second sub-bucket give a worst-of-six p95 of 240,000ms against a merged 60-second p95 of 5,000ms — a 48x understatement of a headline whose tooltip claims it is the worst in the window. Both charts keep the floor, since a readable line was the point of it. Their headlines now come from a second query at the range's natural bucket width, via an optional `readout` on the tile, so each means what its tooltip says regardless of how the plotted buckets are sized. Saturation and backlog are genuinely width-invariant (a max of maxes is the same at any width), so they are unchanged and issue no extra query. Both caught by Devin in review; I had wrongly lumped p95 in with the peaks. **Charts reported a hydration mismatch on every render.** Recharts resolved victory-vendor's CJS entry on the server and its ESM entry in the browser. Those bundle different d3-shape builds, and the CJS one predates d3-path's digit rounding, so every server-rendered curve carried full-precision coordinates while the client rounded to 3 decimals: ``` Server: M0,3C0.9305555555555555,3,1.8611111111111112,3,... Client: M0,3C0.931,3,1.861,3,... ``` Bundling recharts for SSR makes both sides resolve the same ESM build. Verified: 45 of 45 server-rendered chart curves now match the client, and the page loads with an empty console. ## Verification An isolated stack with 40 seeded queues (20 heavily loaded, 20 idle) and 90 minutes of 10-second buckets written into `queue_metrics_raw_v1`, so the real materialized views built `queue_metrics_v1`, `env_metrics_v1` and the 5m rollup. Ground truth for the environment: 260 running against a limit of 300 (**87% saturation**), 800 queued. | | before | after | | -- | -- | -- | | Saturation, page 1 | 82% peak | **87% peak** | | Saturation, page 2 | 5% peak | **87% peak** | | Backlog / delay, page 2 | "No activity" | **800 peak / 59.5s** | | Name search matching nothing | all four charts blank | charts stay environment-wide | | Metric refetches on a page change | 4, each painting a skeleton | **0, no skeleton** | | Buckets drawn as 0ms with no samples | 232 of 349 | **0** | | Throttled readout | 17% | **17%**, unchanged by the wider buckets | | Worst-p95 readout source | plotted buckets | **natural width**, so a sub-minute spike is not averaged away | | Crosshair reach, hovering one detail-page chart | 2 of 4 others | **4 of 4** | | SSR chart curves mismatching the client | 45 | **0** | The bucket floor was measured across ranges: it widens 10s to 60s at 30m and 1h, and is correctly a no-op at 12h (300s) and 7d (3600s). One extra request per page load, for the throttled readout. The built-in Queues dashboard, which reads `env_metrics` independently, agrees at 86.7% and 260 of 300. `internal-packages/tsql` suite green (612 tests), including 5 new ones for the floor that fail without it. Webapp typecheck, oxfmt and oxlint clean. Spot-checked the Run metrics dashboard and the per-queue detail page for SSR regressions from bundling recharts: both render, console clean. The queue detail page carries the same event-driven series, so its scheduling delay, throttling and per-key mean delay take the same treatment. ## Screenshots <img width="2540" height="580" alt="after-page1-charts" src="https://github.com/user-attachments/assets/6cd23f9c-e7fd-4918-bcfa-b1d3340b16d1" /> ## Rollout Already behind the per-organization `queueMetricsUiEnabled` flag, so only gated orgs see any of it. Blast radius is chart values on one page plus the SSR bundling of recharts; rollback is a revert with no data migration. ## Stated limitations - `wait_ms_count` and the quantile state both only count `wait_ms > 0`, so "nothing started in this bucket" and "everything started instantly" are indistinguishable in storage. Both render as a gap. Distinguishing them needs a schema change, which is not in this PR. - The queue name search deliberately no longer narrows the charts. It only did so incidentally and incorrectly before (first 25 matches, and blanked on zero matches). Search-scoped charts would need the full unpaginated matching set and a server-side aggregate; worth its own ticket if we want it. - Bundling recharts for SSR grows the server bundle slightly. That is the cost of both sides resolving one d3-shape build. - The plotted delay line is a smoothed 60-second view, so a sub-minute spike above the one-minute warning threshold can fail to colour the line even though the headline reports it and colours itself. - Every chart inside one synced group shares the floor, because the hover crosshair is a reference line on a category x-axis and only draws where the hovered bucket exists in the other chart's own data. That costs the queue detail page's gauges some resolution (1 minute instead of 10 seconds) in exchange for the crosshair working across the row. Separately, while taking the screenshots I found a pre-existing rendering bug unrelated to this change: a **perfectly flat** saturation series draws no line at all (the readout still shows the right percentage), which looks like the threshold gradient's offset degenerating when the series min equals its max. It reproduces on `main`, so it is not a regression here and I have left it alone; filed as its own issue. Refs TRI-12784 --- .../queues-page-environment-wide-charts.md | 6 + .../components/queues/QueueMetricCards.tsx | 32 ++- .../app/hooks/useMetricResourceQuery.ts | 32 ++- .../route.tsx | 225 ++++++++++-------- .../route.tsx | 26 +- apps/webapp/app/routes/resources.metric.tsx | 3 + .../app/services/queryService.server.ts | 26 +- apps/webapp/vite.config.ts | 2 + .../clickhouse/src/client/tsql.ts | 6 + internal-packages/tsql/src/index.ts | 9 + internal-packages/tsql/src/query/printer.ts | 6 +- .../tsql/src/query/printer_context.ts | 22 +- .../tsql/src/query/time_buckets.test.ts | 48 +++- .../tsql/src/query/time_buckets.ts | 46 +++- 14 files changed, 354 insertions(+), 135 deletions(-) create mode 100644 .server-changes/queues-page-environment-wide-charts.md diff --git a/.server-changes/queues-page-environment-wide-charts.md b/.server-changes/queues-page-environment-wide-charts.md new file mode 100644 index 00000000000..bb3d8ea5da8 --- /dev/null +++ b/.server-changes/queues-page-environment-wide-charts.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +The four charts at the top of the Queues page now always cover the whole environment, so paging through or re-sorting your queues no longer changes them. The scheduling delay chart also leaves a gap where no runs started, instead of dropping to zero. diff --git a/apps/webapp/app/components/queues/QueueMetricCards.tsx b/apps/webapp/app/components/queues/QueueMetricCards.tsx index 95dabff0d71..65865205c55 100644 --- a/apps/webapp/app/components/queues/QueueMetricCards.tsx +++ b/apps/webapp/app/components/queues/QueueMetricCards.tsx @@ -54,6 +54,8 @@ export function useQueueMetric( defaultPeriod?: string; /** Poll ClickHouse on this cadence (ms). Omit to use the query's default interval. */ refreshIntervalMs?: number; + /** Floor for the bucket width, for series too sparse to read at the range's natural width. */ + minBucketSeconds?: number; } ) { return useMetricResourceQuery(query, { @@ -62,6 +64,7 @@ export function useQueueMetric( defaultPeriod: opts.defaultPeriod ?? QUEUE_METRICS_DEFAULT_PERIOD, queues: [opts.queueName], fillGaps: opts.fillGaps, + minBucketSeconds: opts.minBucketSeconds, refreshIntervalMs: opts.refreshIntervalMs, }); } @@ -120,6 +123,14 @@ type QueueMetricChartProps = { /** Reports whether the chart has data to plot (false once it settles on the "no activity" state), * so a wrapping card can hide the legend to match. */ onHasDataChange?: (hasData: boolean) => void; + /** Floor for the bucket width, for series too sparse to read at the range's natural width. */ + minBucketSeconds?: number; + /** + * Column whose value counts the samples behind the plotted series. Where it is zero the metric + * has nothing to report, so every series breaks there instead of reading as a real zero. Keep it + * out of `series` — it is read for this test only, never drawn. + */ + sampleCountColumn?: string; }; // Bare chart (no card chrome) so it can live inside a shared card, e.g. a tabbed panel. @@ -136,6 +147,8 @@ export function QueueMetricChart({ carryBackfill, thresholdStroke, onHasDataChange, + minBucketSeconds, + sampleCountColumn, }: QueueMetricChartProps) { const { rows, showLoading, failed } = useQueueMetric(query, { ids, @@ -143,15 +156,17 @@ export function QueueMetricChart({ queueName, fillGaps, defaultPeriod, + minBucketSeconds, }); const data = useMemo(() => { const points = rows .map((r) => { - const point: { bucket: number } & Record<string, number> = { + const point: { bucket: number } & Record<string, number | null> = { bucket: clickhouseTimeToMs(r.t), }; - for (const s of series) point[s.key] = toNumber(r[s.key]); + const hasSamples = sampleCountColumn ? toNumber(r[sampleCountColumn]) > 0 : true; + for (const s of series) point[s.key] = hasSamples ? toNumber(r[s.key]) : null; return point; }) .filter((p) => Number.isFinite(p.bucket)); @@ -160,7 +175,7 @@ export function QueueMetricChart({ // value and carry it back over the earlier buckets so the line doesn't start at a false 0. if (carryBackfill?.length) { for (const key of carryBackfill) { - const first = points.findIndex((p) => p[key] > 0); + const first = points.findIndex((p) => toNumber(p[key]) > 0); if (first > 0) { const value = points[first]![key]!; for (let i = 0; i < first; i++) points[i]![key] = value; @@ -168,7 +183,7 @@ export function QueueMetricChart({ } } return points; - }, [rows, series, carryBackfill]); + }, [rows, series, carryBackfill, sampleCountColumn]); const chartConfig = useMemo(() => { const cfg: ChartConfig = {}; @@ -205,9 +220,14 @@ export function QueueMetricChart({ // Report data presence so a wrapping card can hide its legend when the chart settles on the // "no activity" state. Only report once loaded, so the legend stays put while loading. + const hasPlottedData = useMemo( + () => data.some((point) => series.some((s) => point[s.key] != null)), + [data, series] + ); + useEffect(() => { - if (!showLoading) onHasDataChange?.(!failed && data.length > 0); - }, [showLoading, failed, data.length, onHasDataChange]); + if (!showLoading) onHasDataChange?.(!failed && hasPlottedData); + }, [showLoading, failed, hasPlottedData, onHasDataChange]); return ( <Chart.Root diff --git a/apps/webapp/app/hooks/useMetricResourceQuery.ts b/apps/webapp/app/hooks/useMetricResourceQuery.ts index a4f26bfd9d2..d219ee73c08 100644 --- a/apps/webapp/app/hooks/useMetricResourceQuery.ts +++ b/apps/webapp/app/hooks/useMetricResourceQuery.ts @@ -21,6 +21,8 @@ export type MetricResourceQueryOptions = { defaultPeriod: string; queues?: string[]; fillGaps?: boolean; + /** Floor for the query's bucket width, for series too sparse to read at the range's width. */ + minBucketSeconds?: number; refreshIntervalMs?: number; }; @@ -49,6 +51,11 @@ function cacheSet(key: string, rows: MetricResourceRow[]) { * back-navigation to the queues list) shows its last data immediately and revalidates in the * background rather than flashing a loading skeleton. */ +/** + * An empty query means the caller has nothing to ask for, so no request is made and any rows or + * failure left by a previous query are dropped — a caller that stops asking must not keep reading + * the last answer, or a stale failure would outlive the query that caused it. + */ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryOptions) { const { organizationId, @@ -56,6 +63,7 @@ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryO environmentId, defaultPeriod, fillGaps, + minBucketSeconds, refreshIntervalMs = 60_000, } = opts; const { period, from, to } = opts.timeRange; @@ -71,10 +79,22 @@ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryO from ?? "", to ?? "", fillGaps ? 1 : 0, + minBucketSeconds ?? "", queuesKey ?? "", query, ].join("|"), - [organizationId, projectId, environmentId, resolvedPeriod, from, to, fillGaps, queuesKey, query] + [ + organizationId, + projectId, + environmentId, + resolvedPeriod, + from, + to, + fillGaps, + minBucketSeconds, + queuesKey, + query, + ] ); const [rows, setRows] = useState<MetricResourceRow[] | null>( @@ -86,6 +106,14 @@ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryO const loadedKeyRef = useRef<string | null>(null); const load = useCallback(() => { + if (!query) { + abortRef.current?.abort(); + loadedKeyRef.current = cacheKey; + setRows(null); + setFailed(false); + setIsLoading(false); + return; + } abortRef.current?.abort(); const controller = new AbortController(); abortRef.current = controller; @@ -112,6 +140,7 @@ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryO organizationId, projectId, environmentId, + ...(minBucketSeconds !== undefined ? { minBucketSeconds } : {}), ...(queuesKey !== undefined ? { queues: queuesKey.split(",") } : {}), }), signal: controller.signal, @@ -142,6 +171,7 @@ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryO from, to, fillGaps, + minBucketSeconds, organizationId, projectId, environmentId, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 47ab9581695..0430058a7af 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -391,13 +391,6 @@ function QueuesWithMetricsView() { const metricsByQueue = metrics?.byQueue ?? {}; - // The four header charts mirror exactly the queue set the table is showing (post-search, - // post-pagination). These are the `task/`-prefixed queue_name values the queue_metrics table - // stores, so the client-side tile queries scope to the same rows the loader listed. - const chartQueueNames = success - ? queues.map((q) => (q.type === "task" ? `task/${q.name}` : q.name)) - : []; - const organization = useOrganization(); const project = useProject(); const env = useEnvironment(); @@ -635,13 +628,7 @@ function QueuesWithMetricsView() { </MetricsLayout.Grid> ) : null} - {/* Env saturation, Backlog, Scheduling delay p95, Throttled viz — full-size, synced, - drag-to-zoom line charts (Agent page pattern). Four chart tiles: 2x2 below lg, 4-up - from lg, derived from the tile count. `kind="charts"` bakes the fixed row height. - Only when there are queues to chart: not-success states (engine-version, no tasks) and a - filtered-to-empty list leave chartQueueNames empty, where the tiles would just render - four "No activity" cards above the blank state. */} - {chartQueueNames.length > 0 ? ( + {success && (hasFilters || totalQueues !== 0) ? ( <ChartSyncProvider onZoom={zoomToTimeFilter}> <MetricsLayout.Grid kind="charts"> {QUEUE_HEADER_TILES.map((tile) => ( @@ -649,7 +636,6 @@ function QueuesWithMetricsView() { key={tile.id} tile={tile} timeRange={timeRange} - queueNames={chartQueueNames} referenceLines={ tile.id === "saturation" ? [ @@ -1202,8 +1188,7 @@ export function QueueFilters() { type MetricTileRow = Record<string, number | string | null>; -/** One charted point per time bucket, already aggregated across the visible queue set. */ -type TilePoint = { bucket: number; value: number }; +type TilePoint = { bucket: number; value: number | null }; // Inline colour swatch matching the chart's warning ("yellow") line — used in tooltip copy that // refers to that colour instead of naming it, so the swatch always matches the chart. @@ -1235,16 +1220,30 @@ type QueueHeaderTile = { /** Hover tooltip explaining the headline readout next to the title (e.g. what "9% of current * period" means). Without it the readout has no tooltip. */ totalTooltip?: string; - // Rows can be one-per-bucket (p95, throttled: aggregated across the set in ClickHouse) or - // one-per-(bucket, queue) (saturation, backlog: summed across the set here, since summing a - // gauge across queues can't be a flat aggregate without double-counting sub-buckets). Either - // way derive returns the per-bucket points the chart draws. + /** Turns one row per bucket into the per-bucket points the chart draws. A null value is a + * bucket the metric has nothing to say about, and the line breaks there rather than reading 0. */ derive: (rows: MetricTileRow[]) => { points: TilePoint[]; total: number; formatTotal?: (total: number) => string; totalClassName?: string; }; + /** + * Optional second query, run at the range's natural bucket width, that owns the headline readout. + * For a headline that is not invariant to bucket width — a share of buckets, or a percentile, + * as opposed to a max over gauges — deriving it from the plotted rows would move the number + * whenever this tile's floor widens them. Only requested while the floor is actually widening + * anything; on ranges whose natural width is already at or above the floor the chart's own rows + * are identical and are reused. + */ + readout?: { + query: string; + derive: (rows: MetricTileRow[]) => { + total: number; + formatTotal?: (total: number) => string; + totalClassName?: string; + }; + }; }; function tileNumber(value: number | string | null): number { @@ -1257,39 +1256,23 @@ function tileTimeToMs(value: number | string | null): number { return Date.parse(s.endsWith("Z") ? s : `${s}Z`); } -// Sums a per-(bucket, queue) row set into one value per bucket. `read` pulls the queue's -// contribution; `envColumn`, when set, carries an env-wide column (identical across the set's -// rows in a bucket) through as the max, so saturation can divide by the env limit. -function sumByBucket( - rows: MetricTileRow[], - read: (row: MetricTileRow) => number, - envColumn?: string -): Array<{ bucket: number; sum: number; env: number }> { - const byBucket = new Map<number, { sum: number; env: number }>(); - for (const row of rows) { - const bucket = tileTimeToMs(row.t); - if (!Number.isFinite(bucket)) continue; - const entry = byBucket.get(bucket) ?? { sum: 0, env: 0 }; - entry.sum += read(row); - if (envColumn) entry.env = Math.max(entry.env, tileNumber(row[envColumn])); - byBucket.set(bucket, entry); - } - return [...byBucket.entries()] - .map(([bucket, { sum, env }]) => ({ bucket, sum, env })) - .sort((a, b) => a.bucket - b.bucket); +/** Peak of a series, ignoring the buckets it has nothing to say about. */ +function peakOf(points: TilePoint[]): number { + return points.reduce((max, p) => (p.value === null ? max : Math.max(max, p.value)), 0); } -// Header tiles fetch their own TRQL query client-side (resources.metric) with fillGaps, scoped to -// the visible queue set (queue_metrics WHERE queue IN <set>). Saturation and backlog GROUP BY the -// queue too and sum here; p95 merges quantile states and throttled sums counters in ClickHouse. +const SCHEDULING_DELAY_QUERY = `SELECT timeBucket() AS t,\n round(quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n sum(wait_ms_count) AS samples\nFROM env_metrics\nGROUP BY t\nORDER BY t`; + +const THROTTLED_QUERY = `SELECT timeBucket() AS t,\n sum(throttled_count) AS throttled\nFROM env_metrics\nGROUP BY t\nORDER BY t`; + const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ { id: "saturation", label: "Env saturation", description: ( <> - How much of the environment's concurrency these queues are using. Turns <WarningSwatch />{" "} - above 100%, when they're into burst capacity. + How much of the environment's concurrency is in use. Turns <WarningSwatch /> above 100%, + when it's into burst capacity. </> ), color: "var(--color-queues)", @@ -1297,35 +1280,32 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ { color: "var(--color-queues)", label: "Saturation" }, { color: "var(--color-warning)", label: "Over limit" }, ], - // Numerator: running summed across the visible set. Denominator: the env-wide limit (same for - // every queue in a bucket), so the line reads as the set's share of the environment capacity. - query: `SELECT timeBucket() AS t,\n queue,\n max(max_running) AS running,\n max(max_env_limit) AS env_limit\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + query: `SELECT timeBucket() AS t,\n max(max_env_running) AS running,\n max(max_env_limit) AS env_limit\nFROM env_metrics\nGROUP BY t\nORDER BY t`, formatValue: (v) => (v > 100 ? `${v}% — over the environment limit` : `${v}%`), formatAxis: (v) => `${v}%`, derive: (rows) => { - const points = sumByBucket(rows, (r) => tileNumber(r.running), "env_limit").map( - ({ bucket, sum, env }) => ({ - bucket, - value: env > 0 ? Math.round((sum / env) * 100) : 0, - }) - ); - const peak = points.reduce((max, p) => Math.max(max, p.value), 0); - return { points, total: peak, formatTotal: (v) => `${v}% peak` }; + const points = rows.map((r) => { + const limit = tileNumber(r.env_limit); + return { + bucket: tileTimeToMs(r.t), + value: limit > 0 ? Math.round((tileNumber(r.running) / limit) * 100) : 0, + }; + }); + return { points, total: peakOf(points), formatTotal: (v) => `${v}% peak` }; }, }, { id: "backlog", label: "Backlog", - description: "How many runs are waiting across these queues, over time.", + description: "How many runs are waiting across the environment, over time.", color: "var(--color-queues)", - query: `SELECT timeBucket() AS t,\n queue,\n max(max_queued) AS queued\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + query: `SELECT timeBucket() AS t,\n max(max_env_queued) AS queued\nFROM env_metrics\nGROUP BY t\nORDER BY t`, derive: (rows) => { - const points = sumByBucket(rows, (r) => tileNumber(r.queued)).map(({ bucket, sum }) => ({ - bucket, - value: sum, + const points = rows.map((r) => ({ + bucket: tileTimeToMs(r.t), + value: tileNumber(r.queued), })); - const peak = points.reduce((max, p) => Math.max(max, p.value), 0); - return { points, total: peak, formatTotal: (v) => `${v.toLocaleString()} peak` }; + return { points, total: peakOf(points), formatTotal: (v) => `${v.toLocaleString()} peak` }; }, }, { @@ -1343,20 +1323,35 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ { color: "var(--color-queues)", label: "p95" }, { color: "var(--color-warning)", label: "Over 1 min" }, ], - // quantilesMerge over the set's rows in a bucket is the true p95 across the union of samples - // (merging quantile states is valid; averaging per-queue percentiles would not be). - query: `SELECT timeBucket() AS t,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95\nFROM queue_metrics\nGROUP BY t\nORDER BY t`, + query: SCHEDULING_DELAY_QUERY, formatValue: formatWaitMs, formatAxis: formatWaitMs, derive: (rows) => { - const points = rows.map((r) => ({ bucket: tileTimeToMs(r.t), value: tileNumber(r.p95) })); - const worst = points.reduce((max, p) => Math.max(max, p.value), 0); - return { - points, - total: worst, - formatTotal: (v) => (v > 0 ? formatWaitMs(v) : "–"), - totalClassName: worst >= 60_000 ? "text-warning" : undefined, - }; + const points = rows.map((r) => ({ + bucket: tileTimeToMs(r.t), + value: tileNumber(r.samples) > 0 ? tileNumber(r.p95) : null, + })); + return { points, total: peakOf(points) }; + }, + readout: { + query: SCHEDULING_DELAY_QUERY, + /** + * Merging quantile states over a wider bucket yields a p95 between the sub-buckets' own, so + * the worst p95 has to be read at the range's natural width or a burst of slow starts shorter + * than the plotted bucket is averaged away. Unlike the gauges, whose max of maxes is the same + * at any width. + */ + derive: (rows) => { + const worst = rows.reduce( + (max, r) => (tileNumber(r.samples) > 0 ? Math.max(max, tileNumber(r.p95)) : max), + 0 + ); + return { + total: worst, + formatTotal: (v) => (v > 0 ? formatWaitMs(v) : "–"), + totalClassName: worst >= 60_000 ? "text-warning" : undefined, + }; + }, }, }, { @@ -1366,32 +1361,41 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ totalTooltip: "The share of the selected window with at least one blocked dequeue.", color: "var(--color-queues)", legend: [{ color: "var(--color-warning)", label: "Throttled" }], - query: `SELECT timeBucket() AS t,\n sum(throttled_count) AS throttled\nFROM queue_metrics\nGROUP BY t\nORDER BY t`, + query: THROTTLED_QUERY, derive: (rows) => { const points = rows.map((r) => ({ bucket: tileTimeToMs(r.t), value: tileNumber(r.throttled), })); - // Share of the window that saw any throttling. A raw event sum isn't interpretable (it - // scales with poll rate and window length); the fraction of buckets with a throttle is. - // The data path fills gaps (zero-fill for this counter), so every bucket in the window is - // present and `points.length` is the honest denominator. - const nonzero = points.filter((p) => p.value > 0).length; - const pct = points.length > 0 ? Math.round((nonzero / points.length) * 100) : 0; - return { - points, - total: pct, - formatTotal: (v) => `${v}% of current period`, - totalClassName: pct > 0 ? "text-warning" : undefined, - }; + return { points, total: peakOf(points) }; + }, + readout: { + query: THROTTLED_QUERY, + /** + * Share of the window that saw any throttling. A raw event sum isn't interpretable (it + * scales with poll rate and window length); the fraction of buckets with a throttle is. + * Gap fill zero-fills this counter, so every bucket in the window is present and the row + * count is the honest denominator. + */ + derive: (rows) => { + const nonzero = rows.filter((r) => tileNumber(r.throttled) > 0).length; + const pct = rows.length > 0 ? Math.round((nonzero / rows.length) * 100) : 0; + return { + total: pct, + formatTotal: (v) => `${v}% of current period`, + totalClassName: pct > 0 ? "text-warning" : undefined, + }; + }, }, }, ]; -// When a search matches no queues the set is empty. We still fetch (hooks can't be conditional), -// but with a queue name that can't exist so the IN filter returns nothing and the tile falls -// through to its "No activity" empty state instead of silently widening to the whole environment. -const NO_QUEUES_SENTINEL = "__no_queues__"; +/** + * Bucket floor shared by every hero tile. Scheduling delay and throttling are event-driven, so at + * the 10-second width a short range would otherwise pick, most buckets hold no samples at all. One + * floor for all four keeps their x-axes identical, which the shared hover crosshair relies on. + */ +const HERO_CHART_MIN_BUCKET_SECONDS = 60; type TileTimeRange = MetricResourceTimeRange; @@ -1401,7 +1405,6 @@ type TileTimeRange = MetricResourceTimeRange; function QueueEnvMetricChart({ tile, timeRange, - queueNames, referenceLines, thresholdStroke, warningOverlay, @@ -1409,8 +1412,6 @@ function QueueEnvMetricChart({ }: { tile: QueueHeaderTile; timeRange: TileTimeRange; - /** The visible queue set (post-search, post-pagination) the chart scopes to. */ - queueNames: string[]; referenceLines?: Array<{ y: number; label?: string; @@ -1427,20 +1428,32 @@ function QueueEnvMetricChart({ const project = useProject(); const environment = useEnvironment(); - // Scope to exactly the queues the table is showing. Empty set => sentinel that matches nothing, - // so the tile shows "No activity" rather than the whole environment. The hook re-fetches when - // this list changes (it keys on the joined names), so search/pagination reflow the charts. - const { rows, showLoading, failed } = useMetricResourceQuery(tile.query, { + const sharedOptions = { organizationId: organization.id, projectId: project.id, environmentId: environment.id, timeRange, defaultPeriod: QUEUE_METRICS_DEFAULT_PERIOD, fillGaps: true, - queues: queueNames.length > 0 ? queueNames : [NO_QUEUES_SENTINEL], + }; + + const { rows, showLoading, failed } = useMetricResourceQuery(tile.query, { + ...sharedOptions, + minBucketSeconds: HERO_CHART_MIN_BUCKET_SECONDS, }); - const { points, total, formatTotal, totalClassName } = tile.derive(rows); + const derived = tile.derive(rows); + const points = derived.points; + + const plottedBucketMs = points.length > 1 ? points[1]!.bucket - points[0]!.bucket : 0; + const floorWidenedBuckets = + plottedBucketMs > 0 && plottedBucketMs <= HERO_CHART_MIN_BUCKET_SECONDS * 1000; + const readoutQuery = tile.readout && floorWidenedBuckets ? tile.readout.query : ""; + const readoutResult = useMetricResourceQuery(readoutQuery, sharedOptions); + + const { total, formatTotal, totalClassName } = tile.readout + ? tile.readout.derive(readoutQuery ? readoutResult.rows : rows) + : derived; // Same point shape the shared axis/tooltip helpers expect. const data = points @@ -1461,14 +1474,16 @@ function QueueEnvMetricChart({ () => buildActivityTimeAxis(data), [data] ); - const hasData = data.length > 0 && data.some((p) => (p[tile.id] as number) > 0); + const hasData = data.length > 0 && data.some((p) => Number(p[tile.id] ?? 0) > 0); // Peak readout lives in the card title (ChartCard has no dedicated value slot). A zero/empty // total renders no readout at all (skipping "0% peak", "0 peak", "0" and the p95 "–" placeholder) // so the card title stands alone until there's a non-zero value to show. - const peak = showLoading ? ( + const readoutLoading = tile.readout ? readoutResult.showLoading : showLoading; + const readoutFailed = tile.readout ? readoutResult.failed : failed; + const peak = readoutLoading ? ( <span className="inline-block h-3 w-12 animate-pulse rounded bg-grid-bright" /> - ) : failed || total === 0 ? null : formatTotal ? ( + ) : readoutFailed || total === 0 ? null : formatTotal ? ( formatTotal(total) ) : ( total.toLocaleString() @@ -1488,7 +1503,7 @@ function QueueEnvMetricChart({ /> </span> {peak != null ? ( - tile.totalTooltip && !showLoading ? ( + tile.totalTooltip && !readoutLoading ? ( <SimpleTooltip button={ <span diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 37f955cd9d5..10f465f82c4 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -191,6 +191,16 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const CK_LIVE_LIMIT = 50; +/** + * Bucket floor for the charts in a synced group. The event-driven series (scheduling delay, + * throttling) need it: their samples only exist when something started or was held back, so at the + * 10-second width a short range picks, most buckets hold nothing and the line reads as a run of + * zeros. The gauges beside them take the same floor because the shared hover crosshair is a + * recharts ReferenceLine on a category x-axis, so it only draws where the hovered bucket + * timestamp exists in the other chart's own data — mixing widths in one group silently drops it. + */ +const SYNCED_CHART_MIN_BUCKET_SECONDS = 60; + // Whole-queue oldest wait right now: for keyed queues the per-key breakdown carries the oldest // enqueue time per key, so the queue's oldest is the max wait across keys; otherwise fall back to // the queue's oldest message directly. Returns null when nothing is waiting. @@ -409,6 +419,7 @@ function OverviewCharts({ className="aspect-[2/1]" query={`SELECT timeBucket() AS t, max(max_running) AS running, max(max_limit) AS limit\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} fillGaps + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} timeRange={timeRange} queueName={queueName} @@ -435,6 +446,7 @@ function OverviewCharts({ className="aspect-[2/1]" query={`SELECT timeBucket() AS t, max(max_queued) AS queued\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} fillGaps + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} timeRange={timeRange} queueName={queueName} @@ -454,6 +466,7 @@ function OverviewCharts({ className="aspect-[2/1]" query={`SELECT timeBucket() AS t,\n deltaSumTimestampMerge(enqueue_delta) AS enqueued,\n deltaSumTimestampMerge(started_delta) AS started\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} fillGaps + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} timeRange={timeRange} queueName={queueName} @@ -470,8 +483,10 @@ function OverviewCharts({ info="How long runs wait before they start." showLegend className="aspect-[2/1]" - query={`SELECT timeBucket() AS t,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[1]) AS p50,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[4]) AS p99\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[1]) AS p50,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[4]) AS p99,\n sum(wait_ms_count) AS samples\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} fillGaps + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} + sampleCountColumn="samples" ids={ids} timeRange={timeRange} queueName={queueName} @@ -493,6 +508,7 @@ function OverviewCharts({ className="aspect-[2/1] sm:col-span-2 sm:aspect-[4/1]" query={`SELECT timeBucket() AS t, sum(throttled_count) AS throttled\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} fillGaps + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} timeRange={timeRange} queueName={queueName} @@ -982,6 +998,7 @@ function KeyDrilldown({ className="aspect-[2/1]" query={`SELECT timeBucket() AS t, max(max_queued) AS queued, max(max_running) AS running\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} fillGaps + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} timeRange={timeRange} queueName={queueName} @@ -995,6 +1012,8 @@ function KeyDrilldown({ title={`Key ${keyName}: throughput`} className="aspect-[2/1]" query={`SELECT timeBucket() AS t, deltaSumTimestampMerge(started_delta) AS started\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} + fillGaps + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} timeRange={timeRange} queueName={queueName} @@ -1003,7 +1022,10 @@ function KeyDrilldown({ <QueueDetailChartCard title={`Key ${keyName}: mean scheduling delay`} className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, if(sum(wait_ms_count) > 0, round(sum(wait_ms_sum) / sum(wait_ms_count)), 0) AS wait\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, if(sum(wait_ms_count) > 0, round(sum(wait_ms_sum) / sum(wait_ms_count)), 0) AS wait, sum(wait_ms_count) AS samples\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} + fillGaps + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} + sampleCountColumn="samples" ids={ids} timeRange={timeRange} queueName={queueName} diff --git a/apps/webapp/app/routes/resources.metric.tsx b/apps/webapp/app/routes/resources.metric.tsx index 1808f6107e4..af98a54b84d 100644 --- a/apps/webapp/app/routes/resources.metric.tsx +++ b/apps/webapp/app/routes/resources.metric.tsx @@ -52,6 +52,7 @@ const MetricWidgetQuery = z.object({ tags: z.array(z.string()).optional(), // Opt into server-side gap fill (carry-forward for gauges, zero-fill for counters). fillGaps: z.boolean().optional(), + minBucketSeconds: z.number().int().positive().max(86_400).optional(), userAuthoredQuery: z.boolean().optional(), }); @@ -89,6 +90,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { providers, tags: _tags, fillGaps, + minBucketSeconds, userAuthoredQuery, } = submission.data; @@ -128,6 +130,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { operations, providers, fillGaps, + minBucketSeconds, userAuthoredQuery, // Set higher concurrency if many widgets are on screen at once customOrgConcurrencyLimit: env.METRIC_WIDGET_DEFAULT_ORG_CONCURRENCY_LIMIT, diff --git a/apps/webapp/app/services/queryService.server.ts b/apps/webapp/app/services/queryService.server.ts index bcceca25bbb..755da8ec3a3 100644 --- a/apps/webapp/app/services/queryService.server.ts +++ b/apps/webapp/app/services/queryService.server.ts @@ -9,8 +9,8 @@ import { import type { CustomerQuerySource } from "@trigger.dev/database"; import { calculateTimeBucketInterval, + intervalToSeconds, type TableSchema, - type TimeBucketInterval, type WhereClauseCondition, } from "@internal/tsql"; import { z } from "zod"; @@ -135,15 +135,6 @@ export function isQueryConcurrencyRejection(error: unknown): boolean { ); } -const INTERVAL_UNIT_SECONDS: Record<TimeBucketInterval["unit"], number> = { - SECOND: 1, - MINUTE: 60, - HOUR: 3_600, - DAY: 86_400, - WEEK: 604_800, - MONTH: 2_592_000, -}; - function floorToSeconds(date: Date, alignSeconds: number): Date { const ms = alignSeconds * 1000; return new Date(Math.floor(date.getTime() / ms) * ms); @@ -167,16 +158,21 @@ function resolveQueryClientType(schema: TableSchema | undefined): ClientType { * rollup's granularity. The rollup has identical logical columns, so only the physical * table (and therefore rows read) changes. */ -function resolveRollup(schema: TableSchema, timeRange: { from: Date; to: Date }): TableSchema { +function resolveRollup( + schema: TableSchema, + timeRange: { from: Date; to: Date }, + minBucketSeconds?: number +): TableSchema { if (!schema.rollups || schema.rollups.length === 0) { return schema; } const interval = calculateTimeBucketInterval( timeRange.from, timeRange.to, - schema.timeBucketThresholds + schema.timeBucketThresholds, + minBucketSeconds ); - const intervalSeconds = interval.value * INTERVAL_UNIT_SECONDS[interval.unit]; + const intervalSeconds = intervalToSeconds(interval); const best = [...schema.rollups] .sort((a, b) => b.minIntervalSeconds - a.minIntervalSeconds) .find((r) => r.minIntervalSeconds <= intervalSeconds); @@ -374,7 +370,9 @@ export async function executeQuery<TOut extends z.ZodSchema>( ); // Serve coarse-bucket queries from the table's rollup when one qualifies. const effectiveSchemas = matchedSchema?.rollups - ? querySchemas.map((s) => (s === matchedSchema ? resolveRollup(s, timeRange) : s)) + ? querySchemas.map((s) => + s === matchedSchema ? resolveRollup(s, timeRange, baseOptions.minBucketSeconds) : s + ) : querySchemas; const queryCacheSettings: ClickHouseSettings = matchedSchema?.queryCache diff --git a/apps/webapp/vite.config.ts b/apps/webapp/vite.config.ts index 0b33a4edb6f..4d7efbfc5ee 100644 --- a/apps/webapp/vite.config.ts +++ b/apps/webapp/vite.config.ts @@ -59,6 +59,8 @@ export default defineConfig({ "@kapaai/react-sdk", "@fingerprintjs/fingerprintjs-pro", "@fingerprintjs/fingerprintjs-pro-spa", + "recharts", + /^victory-vendor/, ], optimizeDeps: { include: ["cron-parser"], diff --git a/internal-packages/clickhouse/src/client/tsql.ts b/internal-packages/clickhouse/src/client/tsql.ts index 2cf586c36f1..f61009237e4 100644 --- a/internal-packages/clickhouse/src/client/tsql.ts +++ b/internal-packages/clickhouse/src/client/tsql.ts @@ -114,6 +114,11 @@ export interface ExecuteTSQLOptions<TOut extends z.ZodSchema> { * (counters zero-fill, gauges carry forward). Off by default. */ fillGaps?: boolean; + /** + * Floor for the `timeBucket()` interval, in seconds. Widens buckets past what the range + * would pick, for series whose samples are too sparse to read at that width. + */ + minBucketSeconds?: number; /** * Set when `query` was written by whoever made the request rather than by us. * A rejection of their SQL is then their mistake, not a bug on our side. @@ -204,6 +209,7 @@ export async function executeTSQL<TOut extends z.ZodSchema>( whereClauseFallback: options.whereClauseFallback, timeRange: options.timeRange, fillGaps: options.fillGaps, + minBucketSeconds: options.minBucketSeconds, }); generatedSql = sql; diff --git a/internal-packages/tsql/src/index.ts b/internal-packages/tsql/src/index.ts index 1ebd1a60a5d..a40ef63a62a 100644 --- a/internal-packages/tsql/src/index.ts +++ b/internal-packages/tsql/src/index.ts @@ -134,7 +134,9 @@ export { // Re-export time bucket utilities export { BUCKET_THRESHOLDS, + INTERVAL_UNIT_SECONDS, calculateTimeBucketInterval, + intervalToSeconds, type BucketThreshold, type TimeBucketInterval, } from "./query/time_buckets.js"; @@ -565,6 +567,12 @@ export interface CompileTSQLOptions { * Off by default; output is unchanged when not set. */ fillGaps?: boolean; + /** + * Floor for the `timeBucket()` interval, in seconds. Widens buckets past what the range + * would pick, for series whose samples are too sparse to read at that width (e.g. a + * percentile over buckets that often contain no samples at all). + */ + minBucketSeconds?: number; } /** @@ -624,6 +632,7 @@ export function compileTSQL(query: string, options: CompileTSQLOptions): PrintRe enforcedWhereClause, timeRange: options.timeRange, fillGaps: options.fillGaps, + minBucketSeconds: options.minBucketSeconds, }); // 6. Print the AST to ClickHouse SQL (enforced conditions applied at printer level) diff --git a/internal-packages/tsql/src/query/printer.ts b/internal-packages/tsql/src/query/printer.ts index ed59b9977fe..adfa62cedb0 100644 --- a/internal-packages/tsql/src/query/printer.ts +++ b/internal-packages/tsql/src/query/printer.ts @@ -624,7 +624,8 @@ export class ClickHousePrinter { const interval = calculateTimeBucketInterval( timeRange.from, timeRange.to, - tableSchema.timeBucketThresholds + tableSchema.timeBucketThresholds, + this.context.minBucketSeconds ); const bucketSql = `toStartOfInterval(${escapeClickHouseIdentifier(clickhouseColumnName)}, INTERVAL ${interval.value} ${interval.unit})`; @@ -3551,7 +3552,8 @@ export class ClickHousePrinter { const interval = calculateTimeBucketInterval( timeRange.from, timeRange.to, - tableSchema.timeBucketThresholds + tableSchema.timeBucketThresholds, + this.context.minBucketSeconds ); // Emit toStartOfInterval(column, INTERVAL N UNIT) diff --git a/internal-packages/tsql/src/query/printer_context.ts b/internal-packages/tsql/src/query/printer_context.ts index a964e2e04af..69fb5b6e944 100644 --- a/internal-packages/tsql/src/query/printer_context.ts +++ b/internal-packages/tsql/src/query/printer_context.ts @@ -128,6 +128,12 @@ export class PrinterContext { /** When true, time-bucketed queries emit rows for empty buckets (opt-in). */ readonly fillGaps?: boolean; + /** + * Floor for the `timeBucket()` interval, in seconds. Widens buckets past what the range + * would pick, for series whose samples are too sparse to read at that width. + */ + readonly minBucketSeconds?: number; + constructor( /** Schema registry containing allowed tables and columns */ public readonly schema: SchemaRegistry, @@ -143,7 +149,9 @@ export class PrinterContext { /** Time range for timeBucket() interval calculation */ timeRange?: TimeRange, /** Opt-in gap-fill for time-bucketed queries */ - fillGaps?: boolean + fillGaps?: boolean, + /** Floor for the timeBucket() interval, in seconds */ + minBucketSeconds?: number ) { // Initialize with default settings this.settings = { ...DEFAULT_QUERY_SETTINGS, ...settings }; @@ -151,6 +159,7 @@ export class PrinterContext { this.enforcedWhereClause = enforcedWhereClause; this.timeRange = timeRange; this.fillGaps = fillGaps; + this.minBucketSeconds = minBucketSeconds; } /** @@ -232,7 +241,8 @@ export class PrinterContext { this.fieldMappings, this.enforcedWhereClause, this.timeRange, - this.fillGaps + this.fillGaps, + this.minBucketSeconds ); // Share the same values map so parameters are unified child.values = this.values; @@ -286,6 +296,11 @@ export interface PrinterContextOptions { timeRange?: TimeRange; /** When true, time-bucketed queries emit rows for empty buckets (opt-in). */ fillGaps?: boolean; + /** + * Floor for the `timeBucket()` interval, in seconds. Widens buckets past what the range + * would pick, for series whose samples are too sparse to read at that width. + */ + minBucketSeconds?: number; } /** @@ -298,6 +313,7 @@ export function createPrinterContext(options: PrinterContextOptions): PrinterCon options.fieldMappings, options.enforcedWhereClause, options.timeRange, - options.fillGaps + options.fillGaps, + options.minBucketSeconds ); } diff --git a/internal-packages/tsql/src/query/time_buckets.test.ts b/internal-packages/tsql/src/query/time_buckets.test.ts index 451e74bccc1..c23fa8a09fd 100644 --- a/internal-packages/tsql/src/query/time_buckets.test.ts +++ b/internal-packages/tsql/src/query/time_buckets.test.ts @@ -1,5 +1,9 @@ import { describe, it, expect } from "vitest"; -import { calculateTimeBucketInterval, type TimeBucketInterval } from "./time_buckets.js"; +import { + calculateTimeBucketInterval, + type BucketThreshold, + type TimeBucketInterval, +} from "./time_buckets.js"; /** * Helper to create a Date range from a start date and a duration @@ -178,4 +182,46 @@ describe("calculateTimeBucketInterval", () => { }); }); }); + + describe("minBucketSeconds floor", () => { + const tenSecondThresholds: BucketThreshold[] = [ + { maxRangeSeconds: 3 * 60 * 60, interval: { value: 10, unit: "SECOND" } }, + ]; + + it("should widen an interval below the floor", () => { + const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 1 * HOUR); + expect( + calculateTimeBucketInterval(from, to, tenSecondThresholds, 60) + ).toEqual<TimeBucketInterval>({ value: 1, unit: "MINUTE" }); + }); + + it("should leave an interval already above the floor alone", () => { + const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 5 * DAY); + expect(calculateTimeBucketInterval(from, to, undefined, 60)).toEqual<TimeBucketInterval>({ + value: 6, + unit: "HOUR", + }); + }); + + it("should be a no-op when no floor is given", () => { + const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 1 * HOUR); + expect( + calculateTimeBucketInterval(from, to, tenSecondThresholds) + ).toEqual<TimeBucketInterval>({ value: 10, unit: "SECOND" }); + }); + + it("should express the floor in the largest unit it divides evenly into", () => { + const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 1 * HOUR); + expect( + calculateTimeBucketInterval(from, to, tenSecondThresholds, 300) + ).toEqual<TimeBucketInterval>({ value: 5, unit: "MINUTE" }); + }); + + it("should fall back to seconds for a floor that is not a whole number of minutes", () => { + const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 1 * HOUR); + expect( + calculateTimeBucketInterval(from, to, tenSecondThresholds, 90) + ).toEqual<TimeBucketInterval>({ value: 90, unit: "SECOND" }); + }); + }); }); diff --git a/internal-packages/tsql/src/query/time_buckets.ts b/internal-packages/tsql/src/query/time_buckets.ts index 04edde007d3..6fbbc1973a9 100644 --- a/internal-packages/tsql/src/query/time_buckets.ts +++ b/internal-packages/tsql/src/query/time_buckets.ts @@ -57,6 +57,36 @@ export const BUCKET_THRESHOLDS: BucketThreshold[] = [ /** Default interval for very large ranges (365+ days) */ const DEFAULT_LARGE_INTERVAL: TimeBucketInterval = { value: 1, unit: "MONTH" }; +/** Seconds in each bucket unit. MONTH is nominal (30 days) and only used for comparisons. */ +export const INTERVAL_UNIT_SECONDS: Record<TimeBucketInterval["unit"], number> = { + SECOND: 1, + MINUTE: 60, + HOUR: 3600, + DAY: 86400, + WEEK: 604800, + MONTH: 2592000, +}; + +/** Duration of a bucket interval, in seconds. */ +export function intervalToSeconds(interval: TimeBucketInterval): number { + return interval.value * INTERVAL_UNIT_SECONDS[interval.unit]; +} + +/** + * Express a duration in seconds as a bucket interval, preferring the largest unit it divides + * evenly into so the emitted `INTERVAL N UNIT` reads naturally (120 -> 2 MINUTE, not 120 SECOND). + */ +function secondsToInterval(seconds: number): TimeBucketInterval { + const units: Array<TimeBucketInterval["unit"]> = ["WEEK", "DAY", "HOUR", "MINUTE"]; + for (const unit of units) { + const unitSeconds = INTERVAL_UNIT_SECONDS[unit]; + if (seconds >= unitSeconds && seconds % unitSeconds === 0) { + return { value: seconds / unitSeconds, unit }; + } + } + return { value: Math.max(1, Math.round(seconds)), unit: "SECOND" }; +} + /** * Calculate the most appropriate time bucket interval for a given time range. * @@ -66,6 +96,9 @@ const DEFAULT_LARGE_INTERVAL: TimeBucketInterval = { value: 1, unit: "MONTH" }; * * @param from - Start of the time range * @param to - End of the time range + * @param thresholds - Table-specific thresholds, defaulting to `BUCKET_THRESHOLDS` + * @param minBucketSeconds - Floor for the returned interval, for series whose samples are too + * sparse to be meaningful at the range's natural bucket width * @returns The recommended bucket interval * * @example @@ -86,10 +119,21 @@ const DEFAULT_LARGE_INTERVAL: TimeBucketInterval = { value: 1, unit: "MONTH" }; export function calculateTimeBucketInterval( from: Date, to: Date, - thresholds?: BucketThreshold[] + thresholds?: BucketThreshold[], + minBucketSeconds?: number ): TimeBucketInterval { const rangeSeconds = Math.abs(to.getTime() - from.getTime()) / 1000; + const interval = pickInterval(rangeSeconds, thresholds); + + if (minBucketSeconds !== undefined && intervalToSeconds(interval) < minBucketSeconds) { + return secondsToInterval(minBucketSeconds); + } + + return interval; +} + +function pickInterval(rangeSeconds: number, thresholds?: BucketThreshold[]): TimeBucketInterval { for (const threshold of thresholds ?? BUCKET_THRESHOLDS) { if (rangeSeconds < threshold.maxRangeSeconds) { return threshold.interval; From 8f9db533507817c923cf0bae38a34ea781c503f3 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:40:41 +0100 Subject: [PATCH 64/67] feat(supervisor): configurable tolerations for run pods (#4491) ## Summary Self-hosted Kubernetes deployments can now add tolerations to run pods, so runs can schedule onto tainted nodes. Previously the only way to do this was to patch the supervisor. `KUBERNETES_RUNNER_TOLERATIONS` takes a comma separated list of `key=value:effect`, or `key:effect` to tolerate any value. It applies to every run pod, and for runs from a schedule tree it merges with the existing `KUBERNETES_SCHEDULED_RUN_TOLERATIONS`. Left unset, nothing changes: no tolerations are added and the pod spec leaves the field off entirely. The Helm chart takes it as a list: ```yaml supervisor: config: kubernetes: runnerTolerations: - dedicated=runs:NoSchedule - spot:NoExecute ``` ## Naming The issue proposed `KUBERNETES_WORKER_TOLERATIONS`. This ships as `KUBERNETES_RUNNER_TOLERATIONS` instead, because `RUNNER_*` is already the prefix for run pod settings (`RUNNER_HEARTBEAT_INTERVAL_SECONDS`, `RUNNER_ADDITIONAL_ENV_VARS`, and `DOCKER_RUNNER_NETWORKS` for the Docker equivalent), whereas "worker" refers to the supervisor itself throughout this app. ## Validation Keys and values are checked against the Kubernetes naming rules when the supervisor starts, so `dedicated=prod runs:NoSchedule` fails immediately with a message naming the offending entry. Without that check a bad value is accepted at startup and then rejected by the API server on every pod create, which stops all runs with the cause buried in an API error. `KUBERNETES_WORKER_NODETYPE_LABEL` is trimmed and validated for the same reason: surrounding whitespace is not valid in a label value, so a padded value fails every pod create today. ## Node selector off switch `KUBERNETES_WORKER_NODETYPE_LABEL` accepts an empty string to skip the node selector entirely, so runs schedule on any node. This already worked and the Helm chart has always shipped it empty, but it was not documented. It is now. The issue also asked for general node affinity configuration. That is not included: the node selector off switch plus tolerations covers the reported problem, and a free form affinity setting is a much larger config surface to commit to. Fixes #4458 --- .../supervisor-run-pod-tolerations.md | 6 + apps/supervisor/src/env.ts | 65 +-------- apps/supervisor/src/envUtil.test.ts | 127 ++++++++++++++++- apps/supervisor/src/envUtil.ts | 130 ++++++++++++++++++ .../src/workloadManager/kubernetes.test.ts | 42 ++++++ .../src/workloadManager/kubernetes.ts | 28 ++-- .../src/workloadManager/kubernetesPodSpec.ts | 30 ++++ docs/self-hosting/env/supervisor.mdx | 3 +- hosting/k8s/helm/templates/supervisor.yaml | 4 + hosting/k8s/helm/values.yaml | 3 +- 10 files changed, 359 insertions(+), 79 deletions(-) create mode 100644 .server-changes/supervisor-run-pod-tolerations.md diff --git a/.server-changes/supervisor-run-pod-tolerations.md b/.server-changes/supervisor-run-pod-tolerations.md new file mode 100644 index 00000000000..885b2b596da --- /dev/null +++ b/.server-changes/supervisor-run-pod-tolerations.md @@ -0,0 +1,6 @@ +--- +area: supervisor +type: feature +--- + +Self-hosted Kubernetes deployments can now add tolerations to run pods, so runs are allowed onto tainted nodes. An invalid toleration now stops the supervisor at startup instead of failing every run pod, so check existing values before upgrading. diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 2d96cb1e407..8184004d9f1 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -1,7 +1,7 @@ import { randomUUID } from "crypto"; import { env as stdEnv } from "std-env"; import { z } from "zod"; -import { AdditionalEnvVars, BoolEnv } from "./envUtil.js"; +import { AdditionalEnvVars, BoolEnv, NodeLabelValue, Tolerations } from "./envUtil.js"; export const Env = z .object({ @@ -173,7 +173,7 @@ export const Env = z // Kubernetes settings KUBERNETES_FORCE_ENABLED: BoolEnv.default(false), KUBERNETES_NAMESPACE: z.string().default("default"), - KUBERNETES_WORKER_NODETYPE_LABEL: z.string().default("v4-worker"), + KUBERNETES_WORKER_NODETYPE_LABEL: NodeLabelValue.default("v4-worker"), KUBERNETES_IMAGE_PULL_SECRETS: z.string().optional(), // csv KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"), KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"), @@ -256,65 +256,8 @@ export const Env = z .max(100) .default(20), - // Schedule toleration settings - scheduled runs tolerate taints on the dedicated pool - // Comma-separated list of tolerations in the format: key=value:effect - // For Exists operator (no value): key:effect - KUBERNETES_SCHEDULED_RUN_TOLERATIONS: z - .string() - .transform((val, ctx) => { - const tolerations = val - .split(",") - .map((entry) => entry.trim()) - .filter((entry) => entry.length > 0) - .map((entry) => { - const colonIdx = entry.lastIndexOf(":"); - if (colonIdx === -1) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: `Invalid toleration format (missing effect): "${entry}"`, - }); - return z.NEVER; - } - - const effect = entry.slice(colonIdx + 1); - const validEffects = ["NoSchedule", "NoExecute", "PreferNoSchedule"]; - if (!validEffects.includes(effect)) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: `Invalid toleration effect "${effect}" in "${entry}". Must be one of: ${validEffects.join( - ", " - )}`, - }); - return z.NEVER; - } - - const keyValue = entry.slice(0, colonIdx); - const eqIdx = keyValue.indexOf("="); - const key = eqIdx === -1 ? keyValue : keyValue.slice(0, eqIdx); - - if (!key) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: `Invalid toleration format (empty key): "${entry}"`, - }); - return z.NEVER; - } - - if (eqIdx === -1) { - return { key, operator: "Exists" as const, effect }; - } - - return { - key, - operator: "Equal" as const, - value: keyValue.slice(eqIdx + 1), - effect, - }; - }); - - return tolerations; - }) - .optional(), + KUBERNETES_RUNNER_TOLERATIONS: Tolerations.optional(), // every run pod + KUBERNETES_SCHEDULED_RUN_TOLERATIONS: Tolerations.optional(), // schedule-tree runs only // Placement tags settings PLACEMENT_TAGS_ENABLED: BoolEnv.default(false), diff --git a/apps/supervisor/src/envUtil.test.ts b/apps/supervisor/src/envUtil.test.ts index c3d35758f16..378830f8ab0 100644 --- a/apps/supervisor/src/envUtil.test.ts +++ b/apps/supervisor/src/envUtil.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { BoolEnv, AdditionalEnvVars } from "./envUtil.js"; +import { BoolEnv, AdditionalEnvVars, NodeLabelValue, Tolerations } from "./envUtil.js"; describe("BoolEnv", () => { it("should parse string 'true' as true", () => { @@ -78,3 +78,128 @@ describe("AdditionalEnvVars", () => { }); }); }); + +describe("NodeLabelValue", () => { + it("should keep a clean value untouched", () => { + expect(NodeLabelValue.parse("v4-worker")).toBe("v4-worker"); + }); + + it("should trim surrounding whitespace, which Kubernetes would reject", () => { + expect(NodeLabelValue.parse(" v4-worker ")).toBe("v4-worker"); + expect(NodeLabelValue.parse("\tv4-worker\n")).toBe("v4-worker"); + }); + + it("should treat a whitespace-only value as the empty off-switch", () => { + expect(NodeLabelValue.parse("")).toBe(""); + expect(NodeLabelValue.parse(" ")).toBe(""); + }); + + it("should still apply a default only when unset", () => { + const withDefault = NodeLabelValue.default("v4-worker"); + expect(withDefault.parse(undefined)).toBe("v4-worker"); + expect(withDefault.parse("")).toBe(""); + }); + + it("should reject a value Kubernetes would reject, rather than 422 every pod create", () => { + for (const invalid of ["my worker", "-bad-", "bad.", "a".repeat(64)]) { + expect(NodeLabelValue.safeParse(invalid).success).toBe(false); + } + }); +}); + +describe("Tolerations", () => { + it("should parse key=value entries as Equal", () => { + expect(Tolerations.parse("dedicated=runs:NoSchedule")).toEqual([ + { key: "dedicated", operator: "Equal", value: "runs", effect: "NoSchedule" }, + ]); + }); + + it("should parse entries without a value as Exists", () => { + expect(Tolerations.parse("scheduled-runs:NoExecute")).toEqual([ + { key: "scheduled-runs", operator: "Exists", effect: "NoExecute" }, + ]); + }); + + it("should keep an empty value as an exact match for a valueless taint", () => { + expect(Tolerations.parse("dedicated=:NoSchedule")).toEqual([ + { key: "dedicated", operator: "Equal", value: "", effect: "NoSchedule" }, + ]); + + expect(Tolerations.parse("dedicated:NoSchedule")).toEqual([ + { key: "dedicated", operator: "Exists", effect: "NoSchedule" }, + ]); + }); + + it("should parse an empty string as no tolerations", () => { + expect(Tolerations.parse("")).toEqual([]); + expect(Tolerations.parse(" ")).toEqual([]); + }); + + it("should skip blank entries and trim whitespace", () => { + expect(Tolerations.parse(" a=b:NoSchedule , ,")).toEqual([ + { key: "a", operator: "Equal", value: "b", effect: "NoSchedule" }, + ]); + }); + + it("should reject a missing effect, an unknown effect, and an empty key", () => { + for (const invalid of ["dedicated=runs", "dedicated=runs:Nope", "=runs:NoSchedule"]) { + expect(Tolerations.safeParse(invalid).success).toBe(false); + } + }); + + it("should accept a hyphenated key, a digit-suffixed key, and every effect", () => { + expect( + Tolerations.parse("capacity-1=true:PreferNoSchedule,spot:NoExecute,gpu=a10:NoSchedule") + ).toEqual([ + { key: "capacity-1", operator: "Equal", value: "true", effect: "PreferNoSchedule" }, + { key: "spot", operator: "Exists", effect: "NoExecute" }, + { key: "gpu", operator: "Equal", value: "a10", effect: "NoSchedule" }, + ]); + }); + + it("should accept a DNS-subdomain prefixed key", () => { + expect( + Tolerations.parse("node.cluster.x-k8s.io/machinepool=scheduled-runs:NoSchedule") + ).toEqual([ + { + key: "node.cluster.x-k8s.io/machinepool", + operator: "Equal", + value: "scheduled-runs", + effect: "NoSchedule", + }, + ]); + }); + + it("should reject a key or value that Kubernetes would reject at pod create", () => { + for (const invalid of [ + "dedicated=prod runs:NoSchedule", + "ded icated=runs:NoSchedule", + "dedicated=-runs:NoSchedule", + `dedicated=${"r".repeat(64)}:NoSchedule`, + `${"a".repeat(64)}=runs:NoSchedule`, + `example.com/${"a".repeat(64)}=runs:NoSchedule`, + "a/b/c=runs:NoSchedule", + "Example.com/pool=runs:NoSchedule", + ]) { + expect(Tolerations.safeParse(invalid).success).toBe(false); + } + }); + + it("should bound the prefix and the name separately, as Kubernetes does", () => { + const longestPrefix = `${"a".repeat(63)}.${"b".repeat(63)}.${"c".repeat(63)}.${"d".repeat(61)}`; + expect(longestPrefix.length).toBe(253); + + expect(Tolerations.parse(`${longestPrefix}/${"n".repeat(63)}=runs:NoSchedule`)).toHaveLength(1); + expect(Tolerations.safeParse(`${longestPrefix}a/pool=runs:NoSchedule`).success).toBe(false); + }); + + it("should tolerate whitespace around the separators", () => { + expect(Tolerations.parse("dedicated = runs : NoSchedule")).toEqual([ + { key: "dedicated", operator: "Equal", value: "runs", effect: "NoSchedule" }, + ]); + }); + + it("should reject a stray extra effect instead of folding it into the value", () => { + expect(Tolerations.safeParse("dedicated=runs:NoSchedule:NoExecute").success).toBe(false); + }); +}); diff --git a/apps/supervisor/src/envUtil.ts b/apps/supervisor/src/envUtil.ts index 917f984cc37..67811f76fcb 100644 --- a/apps/supervisor/src/envUtil.ts +++ b/apps/supervisor/src/envUtil.ts @@ -16,6 +16,136 @@ export const BoolEnv = baseBoolEnv as Omit<typeof baseBoolEnv, "default"> & { default: (value: boolean) => z.ZodDefault<typeof baseBoolEnv>; }; +const QUALIFIED_NAME = /^[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$/; +const DNS_SUBDOMAIN = /^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$/; +const LABEL_VALUE = /^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$/; +const QUALIFIED_NAME_MAX = 63; +const DNS_SUBDOMAIN_MAX = 253; +const LABEL_VALUE_MAX = 63; + +/** + * isLabelValue mirrors the Kubernetes label value rules. Empty is valid upstream. + */ +function isLabelValue(value: string): boolean { + return value.length <= LABEL_VALUE_MAX && LABEL_VALUE.test(value); +} + +/** + * isQualifiedName mirrors the Kubernetes qualified name rules used for taint and + * label keys: an optional DNS subdomain prefix before the slash, then the name. + * The two halves have different length limits and different case rules, so a + * single pattern with one overall bound gets both ends wrong. + */ +function isQualifiedName(key: string): boolean { + const slashIdx = key.indexOf("/"); + + if (slashIdx === -1) { + return key.length <= QUALIFIED_NAME_MAX && QUALIFIED_NAME.test(key); + } + + const prefix = key.slice(0, slashIdx); + const name = key.slice(slashIdx + 1); + + return ( + prefix.length <= DNS_SUBDOMAIN_MAX && + DNS_SUBDOMAIN.test(prefix) && + name.length <= QUALIFIED_NAME_MAX && + QUALIFIED_NAME.test(name) + ); +} + +/** + * A node label value. Trimmed because Kubernetes rejects surrounding whitespace + * outright, so a padded value fails every pod create. Deliberately no `min(1)`: + * empty is the off-switch, and the Helm chart ships empty by default. + */ +export const NodeLabelValue = z.string().trim().refine(isLabelValue, { + message: + "Must be a Kubernetes label value: alphanumeric, with dashes, underscores and dots inside, at most 63 characters", +}); + +/** + * Comma-separated pod tolerations in the format `key=value:effect`, or `key:effect` + * for the Exists operator. Keys and values are checked against the Kubernetes + * naming rules here so a typo fails at startup, rather than 422ing every single + * pod create with the cause buried in an API server message. + */ +export const Tolerations = z.string().transform((val, ctx) => { + return val + .split(",") + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0) + .map((entry) => { + const colonIdx = entry.lastIndexOf(":"); + if (colonIdx === -1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid toleration format (missing effect): "${entry}"`, + }); + return z.NEVER; + } + + const effect = entry.slice(colonIdx + 1).trim(); + const validEffects = ["NoSchedule", "NoExecute", "PreferNoSchedule"]; + if (!validEffects.includes(effect)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid toleration effect "${effect}" in "${entry}". Must be one of: ${validEffects.join( + ", " + )}`, + }); + return z.NEVER; + } + + const keyValue = entry.slice(0, colonIdx); + const eqIdx = keyValue.indexOf("="); + const key = (eqIdx === -1 ? keyValue : keyValue.slice(0, eqIdx)).trim(); + + if (!key) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid toleration format (empty key): "${entry}"`, + }); + return z.NEVER; + } + + if (!isQualifiedName(key)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid toleration key "${key}" in "${entry}". Must be a Kubernetes taint key, optionally prefixed with a DNS subdomain.`, + }); + return z.NEVER; + } + + if (eqIdx === -1) { + return { key, operator: "Exists" as const, effect }; + } + + const value = keyValue.slice(eqIdx + 1).trim(); + if (!value) { + logger.warn( + 'Toleration has an empty value, so it matches only a taint whose value is also empty. Drop the "=" to tolerate any value of this key.', + { entry, key } + ); + } + + if (!isLabelValue(value)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid toleration value "${value}" in "${entry}". Must be a Kubernetes label value: alphanumeric, with dashes, underscores and dots inside.`, + }); + return z.NEVER; + } + + return { + key, + operator: "Equal" as const, + value, + effect, + }; + }); +}); + export const AdditionalEnvVars = z.preprocess((val) => { if (typeof val !== "string") { return val; diff --git a/apps/supervisor/src/workloadManager/kubernetes.test.ts b/apps/supervisor/src/workloadManager/kubernetes.test.ts index 85ad3cbebff..bb15c23e9f4 100644 --- a/apps/supervisor/src/workloadManager/kubernetes.test.ts +++ b/apps/supervisor/src/workloadManager/kubernetes.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from "vitest"; import { BLOCK_IO_URING_SECCOMP_PROFILE, + nodetypeNodeSelector, + runPodTolerations, withBlockIoUringSeccompProfile, } from "./kubernetesPodSpec.js"; @@ -14,6 +16,46 @@ const basePodSpec = { }, }; +describe("nodetypeNodeSelector", () => { + it("omits the nodeSelector entirely when the label is empty or unset", () => { + for (const label of ["", undefined]) { + expect(nodetypeNodeSelector(label)).toEqual({}); + } + }); + + it("pins to nodetype=<label> when set", () => { + expect(nodetypeNodeSelector("v4-worker")).toEqual({ nodeSelector: { nodetype: "v4-worker" } }); + }); +}); + +describe("runPodTolerations", () => { + const worker = [{ key: "dedicated", operator: "Equal", value: "runs", effect: "NoSchedule" }]; + const scheduled = [{ key: "scheduled-runs", operator: "Exists", effect: "NoSchedule" }]; + + it("leaves tolerations unset when neither is configured", () => { + expect(runPodTolerations(undefined, undefined, false)).toBeUndefined(); + expect(runPodTolerations(undefined, undefined, true)).toBeUndefined(); + expect(runPodTolerations([], [], true)).toBeUndefined(); + }); + + it("applies the worker tolerations to every run", () => { + expect(runPodTolerations(worker, undefined, false)).toEqual(worker); + expect(runPodTolerations(worker, undefined, true)).toEqual(worker); + }); + + it("applies the scheduled-run tolerations on their own, as before this option existed", () => { + expect(runPodTolerations(undefined, scheduled, true)).toEqual(scheduled); + expect(runPodTolerations(undefined, scheduled, false)).toBeUndefined(); + expect(runPodTolerations([], scheduled, true)).toEqual(scheduled); + }); + + it("adds the scheduled-run tolerations only for scheduled runs", () => { + expect(runPodTolerations(worker, scheduled, false)).toEqual(worker); + expect(runPodTolerations(worker, [], true)).toEqual(worker); + expect(runPodTolerations(worker, scheduled, true)).toEqual([...worker, ...scheduled]); + }); +}); + describe("withBlockIoUringSeccompProfile", () => { it("adds the Localhost io_uring profile for node-24 and above, preserving pod security defaults", () => { for (const runtime of ["node-24", "node-26", "node-30", "experimental-node-24"]) { diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts index 0263b229e13..1b88bafbc28 100644 --- a/apps/supervisor/src/workloadManager/kubernetes.ts +++ b/apps/supervisor/src/workloadManager/kubernetes.ts @@ -14,7 +14,11 @@ import { PlacementTagProcessor } from "@trigger.dev/core/v3/serverOnly"; import { env } from "../env.js"; import { type K8sApi, createK8sApi, type k8s } from "../clients/kubernetes.js"; import { getRunnerId } from "../util.js"; -import { withBlockIoUringSeccompProfile } from "./kubernetesPodSpec.js"; +import { + nodetypeNodeSelector, + runPodTolerations, + withBlockIoUringSeccompProfile, +} from "./kubernetesPodSpec.js"; type ResourceQuantities = { [K in "cpu" | "memory" | "ephemeral-storage"]?: string; @@ -127,7 +131,7 @@ export class KubernetesWorkloadManager implements WorkloadManager { spec: { ...podSpec, affinity: this.#getAffinity(opts), - tolerations: this.#getScheduleTolerations(this.#isScheduledRun(opts)), + tolerations: this.#getTolerations(this.#isScheduledRun(opts)), terminationGracePeriodSeconds: 60 * 60, containers: [ { @@ -329,13 +333,7 @@ export class KubernetesWorkloadManager implements WorkloadManager { schedulerName: env.KUBERNETES_SCHEDULER_NAME, } : {}), - ...(env.KUBERNETES_WORKER_NODETYPE_LABEL - ? { - nodeSelector: { - nodetype: env.KUBERNETES_WORKER_NODETYPE_LABEL, - }, - } - : {}), + ...nodetypeNodeSelector(env.KUBERNETES_WORKER_NODETYPE_LABEL), ...(env.KUBERNETES_POD_DNS_NDOTS_OVERRIDE_ENABLED ? { dnsConfig: { @@ -557,12 +555,12 @@ export class KubernetesWorkloadManager implements WorkloadManager { }; } - #getScheduleTolerations(isScheduledRun: boolean): k8s.V1Toleration[] | undefined { - if (!isScheduledRun || !env.KUBERNETES_SCHEDULED_RUN_TOLERATIONS?.length) { - return undefined; - } - - return env.KUBERNETES_SCHEDULED_RUN_TOLERATIONS; + #getTolerations(isScheduledRun: boolean): k8s.V1Toleration[] | undefined { + return runPodTolerations( + env.KUBERNETES_RUNNER_TOLERATIONS, + env.KUBERNETES_SCHEDULED_RUN_TOLERATIONS, + isScheduledRun + ); } #getProjectPodAffinity(projectId: string): k8s.V1PodAffinity | undefined { diff --git a/apps/supervisor/src/workloadManager/kubernetesPodSpec.ts b/apps/supervisor/src/workloadManager/kubernetesPodSpec.ts index 32c4410ca43..ba15e563f6d 100644 --- a/apps/supervisor/src/workloadManager/kubernetesPodSpec.ts +++ b/apps/supervisor/src/workloadManager/kubernetesPodSpec.ts @@ -6,6 +6,36 @@ import type { k8s } from "../clients/kubernetes.js"; */ export const BLOCK_IO_URING_SECCOMP_PROFILE = "profiles/block-io-uring.json"; +/** + * An empty label is the documented off-switch, leaving the pod unpinned. The Helm + * chart ships an empty value, so don't collapse this into a fallback default - + * that would pin every chart install to a label its nodes don't carry. + */ +export function nodetypeNodeSelector( + label: string | undefined +): Pick<k8s.V1PodSpec, "nodeSelector"> { + return label ? { nodeSelector: { nodetype: label } } : {}; +} + +/** + * Tolerations for a run pod: the cluster-wide set, plus the scheduled-run set when the + * run came from a schedule tree. Not reconciled - Kubernetes matches tolerations as an + * any-match set, so a broad entry in one set can subsume a narrower one in the other. + * Returns undefined rather than an empty array to leave the field unset. + */ +export function runPodTolerations( + runnerTolerations: k8s.V1Toleration[] | undefined, + scheduledRunTolerations: k8s.V1Toleration[] | undefined, + isScheduledRun: boolean +): k8s.V1Toleration[] | undefined { + const tolerations = [ + ...(runnerTolerations ?? []), + ...(isScheduledRun ? (scheduledRunTolerations ?? []) : []), + ]; + + return tolerations.length > 0 ? tolerations : undefined; +} + /** * Node >= 24 always creates io_uring fds, which can't be checkpointed. Blocking * io_uring_setup makes libuv fall back to epoll. Other runtimes don't need this, diff --git a/docs/self-hosting/env/supervisor.mdx b/docs/self-hosting/env/supervisor.mdx index ffa057747fe..a7e4ef96692 100644 --- a/docs/self-hosting/env/supervisor.mdx +++ b/docs/self-hosting/env/supervisor.mdx @@ -46,7 +46,8 @@ mode: "wide" | **Kubernetes settings** | | | | | `KUBERNETES_FORCE_ENABLED` | No | false | Force Kubernetes mode. | | `KUBERNETES_NAMESPACE` | No | default | The namespace that runs should be in. | -| `KUBERNETES_WORKER_NODETYPE_LABEL` | No | v4-worker | Nodes for runs need this label, e.g. `nodetype=v4-worker`. | +| `KUBERNETES_WORKER_NODETYPE_LABEL` | No | v4-worker | Nodes for runs need `nodetype=<this>`. Empty: any node. | +| `KUBERNETES_RUNNER_TOLERATIONS` | No | — | Run pod tolerations. CSV: `key=value:effect`/`key:effect`. | | `KUBERNETES_IMAGE_PULL_SECRETS` | No | — | Image pull secrets (CSV). | | `KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT` | No | 10Gi | Ephemeral storage size limit. Applies to all runs. | | `KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST` | No | 2Gi | Ephemeral storage size request. Applies to all runs. | diff --git a/hosting/k8s/helm/templates/supervisor.yaml b/hosting/k8s/helm/templates/supervisor.yaml index 11fd7a7f6d9..84a6350f035 100644 --- a/hosting/k8s/helm/templates/supervisor.yaml +++ b/hosting/k8s/helm/templates/supervisor.yaml @@ -170,6 +170,10 @@ spec: value: {{ .Values.supervisor.config.kubernetes.forceEnabled | quote }} - name: KUBERNETES_WORKER_NODETYPE_LABEL value: {{ .Values.supervisor.config.kubernetes.workerNodetypeLabel | quote }} + {{- with .Values.supervisor.config.kubernetes.runnerTolerations }} + - name: KUBERNETES_RUNNER_TOLERATIONS + value: {{ join "," . | quote }} + {{- end }} {{- $registryAuthEnabled := false }} {{- if .Values.registry.deploy }} {{- $registryAuthEnabled = .Values.registry.auth.enabled }} diff --git a/hosting/k8s/helm/values.yaml b/hosting/k8s/helm/values.yaml index e6ea51864f1..354d8e55ba1 100644 --- a/hosting/k8s/helm/values.yaml +++ b/hosting/k8s/helm/values.yaml @@ -296,6 +296,7 @@ supervisor: forceEnabled: true namespace: "" # Default: uses release namespace workerNodetypeLabel: "" # When set, runs will only be scheduled on nodes with "nodetype=<label>" + runnerTolerations: [] # Run pod tolerations, e.g. ["dedicated=runs:NoSchedule"] ephemeralStorageSizeLimit: "" # Default: 10Gi ephemeralStorageSizeRequest: "" # Default: 2Gi´ podCleaner: @@ -386,7 +387,7 @@ supervisor: key: "" nodeSelector: {} - tolerations: [] + tolerations: [] # For the supervisor pod itself, not run pods affinity: {} # PostgreSQL configuration From 3fba04573d7a53cc238f24f5088437c25d031857 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:06:23 +0100 Subject: [PATCH 65/67] fix(supervisor): hold the last backpressure verdict when a read fails (#4444) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dequeue brake released the moment its signal became unreadable. `refresh()` caught any error from `source.read()` and set the verdict to `null`, which `computeEngaged()` treats as not-engaged — so a few failed reads dropped an engaged brake, silently, with no log and no metric. That handling was symmetric while the risk is not. A source that has stopped answering correlates with the pressure the brake exists for, so releasing on read failure gives up protection at exactly the wrong moment; holding too long only costs throughput. Now a failed read keeps the last verdict instead of discarding it. The verdict then ages normally, so the existing `maxVerdictAgeMs` check becomes the grace window and still bounds how long a dead source can hold the brake — a permanently unreachable source releases it rather than pinning dequeuing forever. Because `computeEngaged()` only consults staleness for an *engaged* verdict, a released one is unaffected and stays released. The default grace moves from 15s to 120s, comparable to how long the brake normally stays engaged. One guard worth calling out: holding is only safe when something bounds it, so when `maxVerdictAgeMs` is unset the previous discard behaviour is kept. Otherwise an unbounded hold could pin the brake indefinitely. Read failures were previously invisible — the catch block neither logged nor counted. Adds a `read_failures_total` counter, plus an error log on the transition into failure rather than once per tick, since the refresh loop runs every second. The post-release ramp needs no change: it anchors off the engaged-to-released transition, so a grace-window release still ramps back up instead of snapping to full rate, which is what you want after a blind period. Tests cover holding while reads fail, releasing past the max age, and the existing unbounded-config paths are unchanged. --- .../backpressure-hold-last-verdict.md | 6 ++ .../src/backpressure/backpressureMetrics.ts | 8 +++ .../backpressure/backpressureMonitor.test.ts | 55 +++++++++++++++++++ .../src/backpressure/backpressureMonitor.ts | 43 +++++++++++---- apps/supervisor/src/env.ts | 2 +- 5 files changed, 103 insertions(+), 11 deletions(-) create mode 100644 .server-changes/backpressure-hold-last-verdict.md diff --git a/.server-changes/backpressure-hold-last-verdict.md b/.server-changes/backpressure-hold-last-verdict.md new file mode 100644 index 00000000000..18f3ca7fc50 --- /dev/null +++ b/.server-changes/backpressure-hold-last-verdict.md @@ -0,0 +1,6 @@ +--- +area: supervisor +type: fix +--- + +When the capacity signal drops out, the last decision is held for a grace period rather than released. diff --git a/apps/supervisor/src/backpressure/backpressureMetrics.ts b/apps/supervisor/src/backpressure/backpressureMetrics.ts index ffe57628548..9b622357a38 100644 --- a/apps/supervisor/src/backpressure/backpressureMetrics.ts +++ b/apps/supervisor/src/backpressure/backpressureMetrics.ts @@ -8,6 +8,8 @@ export class BackpressureMetrics { readonly dryRun: Gauge<string>; /** Dequeue attempts the gate skipped - or would have, in dry-run (labelled). */ readonly skipsTotal: Counter<string>; + /** Verdict source reads that failed (threw). */ + readonly readFailuresTotal: Counter<string>; constructor(opts: { register: Registry; prefix?: string }) { const prefix = opts.prefix ?? "supervisor_backpressure"; @@ -30,5 +32,11 @@ export class BackpressureMetrics { labelNames: ["dry_run"], registers: [opts.register], }); + + this.readFailuresTotal = new Counter({ + name: `${prefix}_read_failures_total`, + help: "Verdict source reads that threw", + registers: [opts.register], + }); } } diff --git a/apps/supervisor/src/backpressure/backpressureMonitor.test.ts b/apps/supervisor/src/backpressure/backpressureMonitor.test.ts index 7af28ffc9f5..e6c00394bbb 100644 --- a/apps/supervisor/src/backpressure/backpressureMonitor.test.ts +++ b/apps/supervisor/src/backpressure/backpressureMonitor.test.ts @@ -89,6 +89,60 @@ describe("BackpressureMonitor", () => { monitor.stop(); }); + it("holds an engaged verdict while reads fail, then releases past the max age", async () => { + let call = 0; + const source: BackpressureSignalSource = { + read: async () => { + call++; + if (call === 1) { + return { engaged: true, ts: Date.now() }; + } + throw new Error("signal source unreachable"); + }, + }; + const monitor = new BackpressureMonitor({ + enabled: true, + source, + refreshIntervalMs: 1000, + maxVerdictAgeMs: 15_000, + }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + expect(monitor.shouldSkipDequeue()).toBe(true); + + await vi.advanceTimersByTimeAsync(5000); + expect(monitor.shouldSkipDequeue()).toBe(true); // read failing, verdict held + + await vi.advanceTimersByTimeAsync(11_000); + expect(monitor.shouldSkipDequeue()).toBe(false); // past max age, released + + monitor.stop(); + }); + + it("releases immediately on an explicit null even when a grace window is configured", async () => { + let engaged: boolean | null = true; + const source: BackpressureSignalSource = { + read: async () => (engaged === null ? null : { engaged, ts: Date.now() }), + }; + const monitor = new BackpressureMonitor({ + enabled: true, + source, + refreshIntervalMs: 1000, + maxVerdictAgeMs: 15_000, + }); + + monitor.start(); + await vi.advanceTimersByTimeAsync(0); + expect(monitor.shouldSkipDequeue()).toBe(true); + + engaged = null; + await vi.advanceTimersByTimeAsync(1000); + expect(monitor.shouldSkipDequeue()).toBe(false); // null is an answer, not a failure + + monitor.stop(); + }); + it("fails open when the source reports unknown (null)", async () => { const { source } = countingSource(null); const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 }); @@ -292,6 +346,7 @@ describe("BackpressureMonitor", () => { const logs: Array<{ message: string; meta?: Record<string, unknown> }> = []; const logger = { info: (message: string, meta?: Record<string, unknown>) => logs.push({ message, meta }), + error: (message: string, meta?: Record<string, unknown>) => logs.push({ message, meta }), }; const monitor = new BackpressureMonitor({ enabled: true, diff --git a/apps/supervisor/src/backpressure/backpressureMonitor.ts b/apps/supervisor/src/backpressure/backpressureMonitor.ts index 6b4170697e5..aa16fdeaa60 100644 --- a/apps/supervisor/src/backpressure/backpressureMonitor.ts +++ b/apps/supervisor/src/backpressure/backpressureMonitor.ts @@ -2,6 +2,7 @@ import type { BackpressureMetrics } from "./backpressureMetrics.js"; export interface BackpressureLogger { info(message: string, meta?: Record<string, unknown>): void; + error(message: string, meta?: Record<string, unknown>): void; } export type BackpressureVerdict = { @@ -11,9 +12,10 @@ export type BackpressureVerdict = { }; /** - * Source of the current backpressure verdict. `read()` returns `null` when the - * verdict is unknown (missing/unreadable) - the monitor treats unknown as - * "not engaged" (fail-open). + * Source of the current backpressure verdict. `read()` returns `null` when the source + * answered but there is no verdict - the monitor treats that as "not engaged" + * (fail-open). A thrown error is different: the read itself failed, so the monitor + * keeps the previous verdict until it ages past `maxVerdictAgeMs`. */ export interface BackpressureSignalSource { read(): Promise<BackpressureVerdict | null>; @@ -24,8 +26,9 @@ export type BackpressureMonitorOptions = { source: BackpressureSignalSource; refreshIntervalMs?: number; /** - * If set, a cached verdict older than this is treated as unknown (fail-open). - * Guards against the source silently going stale (e.g. hanging reads). + * If set, an engaged verdict older than this is released (fail-open), bounding how + * long a dead source can hold the brake. Reads that fail keep the last verdict, so + * this doubles as the grace window for riding out a transient source outage. */ maxVerdictAgeMs?: number; /** @@ -54,6 +57,7 @@ export class BackpressureMonitor { private refreshInFlight = false; private wasEngaged = false; private releasedAt?: number; + private readFailing = false; constructor(private readonly opts: BackpressureMonitorOptions) { this.opts.metrics?.dryRun.set(this.opts.dryRun ? 1 : 0); @@ -152,12 +156,31 @@ export class BackpressureMonitor { } private async refresh(): Promise<void> { + let next: BackpressureVerdict | null = null; + let readError: unknown; try { - this.verdict = await this.opts.source.read(); - } catch { - // Fail-open: a dead/unreachable source must never pin the brake. Treat as - // unknown (no verdict) so dequeue resumes as if backpressure were off. - this.verdict = null; + next = await this.opts.source.read(); + } catch (error) { + readError = error; + } + + if (readError === undefined) { + this.verdict = next; // an explicit null means "no pressure", so honour it + this.readFailing = false; + } else { + const held = this.opts.maxVerdictAgeMs !== undefined; + if (!held) { + this.verdict = null; // unbounded hold could pin the brake forever + } + this.opts.metrics?.readFailuresTotal.inc(); + if (!this.readFailing) { + this.readFailing = true; // log once per outage, not once per tick + this.opts.logger?.error("backpressure read failed", { + reason: String(readError), + heldPreviousVerdict: held, + engaged: this.computeEngaged(), + }); + } } // Track the engaged→released transition to anchor the resume ramp. Use the diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 8184004d9f1..670a00ab582 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -79,7 +79,7 @@ export const Env = z .number() .int() .positive() - .default(15_000), // Stale verdict → fail-open (treat as not engaged) + .default(120_000), // Grace window: held verdict older than this → fail-open TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST: z.string().optional(), TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PORT: z.coerce.number().int().optional(), TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_USERNAME: z.string().optional(), From 57254b57fbc46f2f524541c8a29d8aee17fa7b5b Mon Sep 17 00:00:00 2001 From: Eric Allam <eric@trigger.dev> Date: Mon, 3 Aug 2026 17:09:03 +0100 Subject: [PATCH 66/67] fix(webapp): make prop-types a production dependency (#4492) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary The webapp's server bundle imports `prop-types` directly, but the package was declared only as a `devDependency`. A production install therefore leaves it out and the built server fails to boot: ``` Failed to start server: Error [ERR_MODULE_NOT_FOUND]: Cannot find package 'prop-types' imported from /triggerdotdev/apps/webapp/build/server/assets/server-build-*.js ``` Moving it to `dependencies` is the whole change. ## Why the bundle imports it Nothing in the webapp's own code uses `prop-types` — there is no reference to it, or to `PropTypes`, anywhere under `apps/webapp/app`. It arrives through `recharts`, whose `react-smooth` dependency still declares `propTypes` on its components. That was invisible until recently. While `recharts` was resolved at runtime, its `prop-types` import was satisfied inside `recharts`' own dependency tree, which is production all the way down. #4486 added `recharts` and `victory-vendor` to `ssr.noExternal` to fix a hydration mismatch on every server-rendered chart; that inlines `react-smooth` into the server bundle, which moves its `prop-types` import into the webapp's own resolution scope — where the package was not available in production. So the bundling change was correct about *which* d3-shape build both sides resolve, and wrong about what the production runtime would be able to find. ## Verification `docker/Dockerfile` builds the runtime dependencies with `pnpm install --prod` against a `turbo prune --scope=webapp --docker` output, so I reproduced exactly that: pruned the workspace, installed with `--prod`, and imported `prop-types` from `apps/webapp`. | | result | | -- | -- | | `main` as it stands (devDependency only) | `FAILS: ERR_MODULE_NOT_FOUND` | | with this change | `prop-types resolves OK` | It resolves both as a CommonJS `require` and as an ESM `import`, which is the form the bundle uses. I also checked this is not one symptom of a wider problem: of the 169 bare specifier roots the server bundle imports, `prop-types` is the **only** one that is a devDependency and not a production dependency. The rest are node builtins or production dependencies. The hydration fix from #4486 is unaffected — the rebuilt bundle still carries the rounding d3-path build. ## Notes `prop-types` is inert in production (its entry point swaps in `factoryWithThrowingShims`), so this adds a 124 KB package that does no work at runtime. It has to be resolvable regardless, because the import is real. An alternative would be adding `prop-types` to `ssr.noExternal` so it is inlined and needs no runtime resolution. That keeps the dependency list honest about the fact that the webapp itself does not use it, at the cost of bundling a CommonJS package into the ESM server output. This route is the smaller, better-understood change. Worth following up separately: a check that every bare import in the server bundle resolves from a production install would have caught this before it landed. Local development installs every devDependency, so the gap is invisible when the built server is run from a working tree. --- apps/webapp/package.json | 2 +- pnpm-lock.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 849db7fc773..42900ea2f55 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -179,6 +179,7 @@ "prism-react-renderer": "^2.3.1", "prismjs": "^1.30.0", "prom-client": "^15.1.0", + "prop-types": "^15.8.1", "qrcode.react": "^4.2.0", "random-words": "^2.0.0", "react": "^18.2.0", @@ -260,7 +261,6 @@ "npm-run-all": "^4.1.5", "postcss-import": "^16.0.1", "postcss-loader": "^8.1.1", - "prop-types": "^15.8.1", "rimraf": "^6.0.1", "style-loader": "^3.3.4", "supertest": "^7.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f37e3250982..45163c161bb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -668,6 +668,9 @@ importers: prom-client: specifier: ^15.1.0 version: 15.1.0 + prop-types: + specifier: ^15.8.1 + version: 15.8.1 qrcode.react: specifier: ^4.2.0 version: 4.2.0(react@18.3.1) @@ -906,9 +909,6 @@ importers: postcss-loader: specifier: ^8.1.1 version: 8.1.1(postcss@8.5.15)(typescript@6.0.3)(webpack@5.102.1(@swc/core@1.3.26)(esbuild@0.15.18)) - prop-types: - specifier: ^15.8.1 - version: 15.8.1 rimraf: specifier: ^6.0.1 version: 6.0.1 From fbd6df33b4cdecfc57b26e05792150742a709b70 Mon Sep 17 00:00:00 2001 From: Katia Bulatova <katia@trigger.dev> Date: Mon, 3 Aug 2026 19:29:33 +0200 Subject: [PATCH 67/67] feat(webapp): Themes + contrast settings update (#4206) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds System Preferences, Dark and Light themes, gated by the `hasThemeSwitcher` feature flag (off by default — dark stays the default theme for everyone). Old theme is now "Classic"and set as default. "System preferences" theme has both Light and Dark modes and uses your laptop settings to use a correct one. It has less color accents (specifically less colored text), and they are the same for both modes, only grayscale values change between them. And Light/Dark themes can be used separately. New Contrast setting is available for System Preferences, Dark and Light themes - it changes the contrast for the whole app. All new visual Settings live in Account. --- .server-changes/light-theme.md | 6 + apps/webapp/app/assets/icons/MachineIcon.tsx | 33 +- ...dashboard-background-menu-bottom-light.jpg | Bin 0 -> 790 bytes ...ed-dashboard-background-menu-top-light.jpg | Bin 0 -> 3588 bytes ...urred-dashboard-background-table-light.jpg | Bin 0 -> 18021 bytes apps/webapp/app/components/AskAI.tsx | 2 +- .../app/components/BackgroundWrapper.tsx | 90 +- .../app/components/BlankStatePanels.tsx | 3 +- .../components/BulkActionFilterSummary.tsx | 4 +- apps/webapp/app/components/DevPresence.tsx | 6 +- apps/webapp/app/components/ErrorDisplay.tsx | 6 +- apps/webapp/app/components/FeatureBadges.tsx | 24 +- .../webapp/app/components/LoginPageLayout.tsx | 2 +- .../app/components/admin/debugTooltip.tsx | 2 +- .../billing/AnimatedOrgBannerBar.tsx | 9 +- .../billing/BillingLimitRecoveryPanel.tsx | 2 +- .../app/components/billing/OrgBanner.tsx | 10 +- .../app/components/billing/UsageBar.tsx | 2 +- .../app/components/code/ChartConfigPanel.tsx | 4 +- .../webapp/app/components/code/TSQLEditor.tsx | 2 +- .../app/components/code/codeMirrorTheme.ts | 8 +- .../environments/EnvironmentLabel.tsx | 14 +- .../errors/ConfigureErrorAlerts.tsx | 11 +- .../components/errors/ErrorStatusBadge.tsx | 19 +- .../integrations/VercelOnboardingModal.tsx | 4 +- .../components/navigation/SideMenuHeader.tsx | 14 +- .../components/navigation/SideMenuItem.tsx | 5 +- .../components/navigation/TreeConnectors.tsx | 4 +- .../components/primitives/AppliedFilter.tsx | 4 +- .../app/components/primitives/Avatar.tsx | 20 +- .../app/components/primitives/Badge.tsx | 6 +- .../components/primitives/BreadcrumbIcon.tsx | 2 +- .../app/components/primitives/Buttons.tsx | 24 +- .../app/components/primitives/Checkbox.tsx | 14 +- .../app/components/primitives/ClientTabs.tsx | 6 +- .../components/primitives/ClipboardField.tsx | 8 +- .../app/components/primitives/DateField.tsx | 2 +- .../app/components/primitives/DateTime.tsx | 5 +- .../app/components/primitives/Input.tsx | 8 +- .../primitives/InputNumberStepper.tsx | 6 +- .../app/components/primitives/Popover.tsx | 19 +- .../app/components/primitives/RadioButton.tsx | 15 +- .../app/components/primitives/Resizable.tsx | 32 +- .../primitives/SegmentedControl.tsx | 5 +- .../app/components/primitives/Select.tsx | 3 +- .../app/components/primitives/Slider.tsx | 23 +- .../app/components/primitives/Switch.tsx | 4 +- .../components/primitives/UsageSparkline.tsx | 2 + .../components/primitives/charts/Chart.tsx | 4 +- .../primitives/charts/ChartZoom.tsx | 8 +- .../app/components/query/QueryEditor.tsx | 2 +- .../app/components/queues/QueueControls.tsx | 23 +- .../components/queues/QueueMetricCards.tsx | 6 +- .../app/components/runs/v3/AIFilterInput.tsx | 2 +- .../app/components/runs/v3/BatchStatus.tsx | 7 +- .../app/components/runs/v3/BulkAction.tsx | 7 +- .../components/runs/v3/DeploymentStatus.tsx | 3 +- .../app/components/runs/v3/RunFilters.tsx | 10 +- apps/webapp/app/components/runs/v3/RunTag.tsx | 31 +- .../app/components/runs/v3/SpanEvents.tsx | 2 +- .../app/components/runs/v3/SpanTitle.tsx | 3 +- .../runs/v3/TaskRunAttemptStatus.tsx | 5 +- .../app/components/runs/v3/TaskRunStatus.tsx | 7 +- .../components/runs/v3/WaitpointStatus.tsx | 5 +- .../runs/v3/agent/AgentMessageView.tsx | 4 +- .../components/runs/v3/ai/AIChatMessages.tsx | 2 +- .../components/runs/v3/ai/AIModelSummary.tsx | 2 +- .../components/runs/v3/ai/SpanMetricRow.tsx | 2 +- .../components/sessions/v1/SessionStatus.tsx | 7 +- apps/webapp/app/hooks/useSystemThemeSync.ts | 29 + apps/webapp/app/hooks/useThemeColor.ts | 30 +- apps/webapp/app/root.tsx | 58 +- .../route.tsx | 2 +- .../route.tsx | 4 +- .../route.tsx | 2 +- .../route.tsx | 2 +- .../route.tsx | 6 +- .../route.tsx | 53 +- .../route.tsx | 19 +- .../route.tsx | 30 +- .../TableSchemaContent.tsx | 2 +- .../route.tsx | 55 +- .../route.tsx | 4 +- .../route.tsx | 1 + .../route.tsx | 4 +- .../route.tsx | 6 +- .../route.tsx | 25 +- .../route.tsx | 4 +- .../route.tsx | 2 +- .../webapp/app/routes/_app.orgs.new/route.tsx | 5 +- .../app/routes/account._index/route.tsx | 178 +- apps/webapp/app/routes/invites.tsx | 6 +- ...cts.$projectParam.env.$envParam.github.tsx | 7 +- .../route.tsx | 10 +- ...cts.$projectParam.env.$envParam.vercel.tsx | 10 +- ...ces.orgs.$organizationSlug.select-plan.tsx | 2 +- .../services/dashboardPreferences.server.ts | 149 +- apps/webapp/app/tailwind.css | 346 +- apps/webapp/app/utils/dashboardPreferences.ts | 92 + apps/webapp/app/utils/themePreference.ts | 25 + apps/webapp/app/v3/featureFlags.server.ts | 26 + apps/webapp/app/v3/featureFlags.ts | 3 + apps/webapp/package.json | 1 + apps/webapp/seed-queue-metrics.mts | 991 +++ apps/webapp/test/themePreference.test.ts | 53 + .../llm-model-catalog/src/defaultPrices.ts | 6169 ++++++++--------- .../llm-model-catalog/src/modelCatalog.ts | 4059 ++++++----- 107 files changed, 7792 insertions(+), 5314 deletions(-) create mode 100644 .server-changes/light-theme.md create mode 100644 apps/webapp/app/assets/images/blurred-dashboard-background-menu-bottom-light.jpg create mode 100644 apps/webapp/app/assets/images/blurred-dashboard-background-menu-top-light.jpg create mode 100644 apps/webapp/app/assets/images/blurred-dashboard-background-table-light.jpg create mode 100644 apps/webapp/app/hooks/useSystemThemeSync.ts create mode 100644 apps/webapp/app/utils/dashboardPreferences.ts create mode 100644 apps/webapp/app/utils/themePreference.ts create mode 100644 apps/webapp/seed-queue-metrics.mts create mode 100644 apps/webapp/test/themePreference.test.ts diff --git a/.server-changes/light-theme.md b/.server-changes/light-theme.md new file mode 100644 index 00000000000..39d4f5892d2 --- /dev/null +++ b/.server-changes/light-theme.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: feature +--- + +The dashboard gets a refreshed Dark theme as its new default look. On the account page you can switch to Light, System preference (follows your OS), or the original Classic look, and fine-tune the new themes with a contrast slider. diff --git a/apps/webapp/app/assets/icons/MachineIcon.tsx b/apps/webapp/app/assets/icons/MachineIcon.tsx index f07e7467b0d..896ccdfbc5f 100644 --- a/apps/webapp/app/assets/icons/MachineIcon.tsx +++ b/apps/webapp/app/assets/icons/MachineIcon.tsx @@ -48,20 +48,25 @@ export function MachineDefaultIcon({ className }: { className?: string }) { function MachineIconNoMachine({ className }: { className?: string }) { return ( - <svg className={className} viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg"> + <svg + className={cn("text-[#D7D9DD] light:text-charcoal-500", className)} + viewBox="0 0 20 20" + fill="none" + xmlns="http://www.w3.org/2000/svg" + > <path d="M16.1875 13C16.6362 13 17 13.3638 17 13.8125V16.1875C17 16.6362 16.6362 17 16.1875 17H6.18945C6.25882 16.9453 6.32663 16.8872 6.39062 16.8232L10.2139 13H16.1875Z" - fill="#D7D9DD" + fill="currentColor" fillOpacity="0.4" /> <path d="M4.84961 12H3.8125C3.36377 12 3 11.6362 3 11.1875V8.8125C3 8.36377 3.36377 8 3.8125 8H8.84961L4.84961 12ZM16.1875 8C16.6362 8 17 8.36377 17 8.8125V11.1875C17 11.6362 16.6362 12 16.1875 12H11.2139L15.2139 8H16.1875Z" - fill="#D7D9DD" + fill="currentColor" fillOpacity="0.4" /> <path d="M9.85059 7H3.8125C3.36377 7 3 6.63623 3 6.1875V3.8125C3 3.36377 3.36377 3 3.8125 3H13.8506L9.85059 7Z" - fill="#D7D9DD" + fill="currentColor" fillOpacity="0.4" /> <line @@ -69,7 +74,7 @@ function MachineIconNoMachine({ className }: { className?: string }) { y1="16.9998" x2="17.0018" y2="2.99805" - stroke="#D7D9DD" + stroke="currentColor" strokeOpacity="0.4" strokeWidth="2" strokeLinecap="round" @@ -83,17 +88,17 @@ function MachineIconMicro({ className }: { className?: string }) { <svg className={className} viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg"> <path d="M16.1875 13C16.6362 13 17 13.3638 17 13.8125V16.1875C17 16.6362 16.6362 17 16.1875 17H3.8125C3.36377 17 3 16.6362 3 16.1875V13.8125C3 13.3638 3.36377 13 3.8125 13H16.1875ZM5 14C4.44772 14 4 14.4477 4 15C4 15.5523 4.44772 16 5 16C5.55228 16 6 15.5523 6 15C6 14.4477 5.55228 14 5 14Z" - fill="#D7D9DD" + fill="currentColor" fillOpacity="0.4" /> <path d="M16.1875 8C16.6362 8 17 8.36377 17 8.8125V11.1875C17 11.6362 16.6362 12 16.1875 12H3.8125C3.36377 12 3 11.6362 3 11.1875V8.8125C3 8.36377 3.36377 8 3.8125 8H16.1875ZM5 9C4.44772 9 4 9.44772 4 10C4 10.5523 4.44772 11 5 11C5.55228 11 6 10.5523 6 10C6 9.44772 5.55228 9 5 9Z" - fill="#D7D9DD" + fill="currentColor" fillOpacity="0.4" /> <path d="M16.1875 3C16.6362 3 17 3.36377 17 3.8125V6.1875C17 6.63623 16.6362 7 16.1875 7H3.8125C3.36377 7 3 6.63623 3 6.1875V3.8125C3 3.36377 3.36377 3 3.8125 3H16.1875ZM5 4C4.44772 4 4 4.44772 4 5C4 5.55228 4.44772 6 5 6C5.55228 6 6 5.55228 6 5C6 4.44772 5.55228 4 5 4Z" - fill="#D7D9DD" + fill="currentColor" fillOpacity="0.4" /> </svg> @@ -109,12 +114,12 @@ function MachineIconSmall1x({ className }: { className?: string }) { /> <path d="M16.1875 8C16.6362 8 17 8.36377 17 8.8125V11.1875C17 11.6362 16.6362 12 16.1875 12H3.8125C3.36377 12 3 11.6362 3 11.1875V8.8125C3 8.36377 3.36377 8 3.8125 8H16.1875ZM5 9C4.44772 9 4 9.44772 4 10C4 10.5523 4.44772 11 5 11C5.55228 11 6 10.5523 6 10C6 9.44772 5.55228 9 5 9Z" - fill="#D7D9DD" + fill="currentColor" fillOpacity="0.4" /> <path d="M16.1875 3C16.6362 3 17 3.36377 17 3.8125V6.1875C17 6.63623 16.6362 7 16.1875 7H3.8125C3.36377 7 3 6.63623 3 6.1875V3.8125C3 3.36377 3.36377 3 3.8125 3H16.1875ZM5 4C4.44772 4 4 4.44772 4 5C4 5.55228 4.44772 6 5 6C5.55228 6 6 5.55228 6 5C6 4.44772 5.55228 4 5 4Z" - fill="#D7D9DD" + fill="currentColor" fillOpacity="0.4" /> </svg> @@ -130,12 +135,12 @@ function MachineIconSmall2x({ className }: { className?: string }) { /> <path d="M16.1875 8C16.6362 8 17 8.36377 17 8.8125V11.1875C17 11.6362 16.6362 12 16.1875 12H3.8125C3.36377 12 3 11.6362 3 11.1875V8.8125C3 8.36377 3.36377 8 3.8125 8H16.1875ZM5 9C4.44772 9 4 9.44772 4 10C4 10.5523 4.44772 11 5 11C5.55228 11 6 10.5523 6 10C6 9.44772 5.55228 9 5 9Z" - fill="#D7D9DD" + fill="currentColor" fillOpacity="0.4" /> <path d="M16.1875 3C16.6362 3 17 3.36377 17 3.8125V6.1875C17 6.63623 16.6362 7 16.1875 7H3.8125C3.36377 7 3 6.63623 3 6.1875V3.8125C3 3.36377 3.36377 3 3.8125 3H16.1875ZM5 4C4.44772 4 4 4.44772 4 5C4 5.55228 4.44772 6 5 6C5.55228 6 6 5.55228 6 5C6 4.44772 5.55228 4 5 4Z" - fill="#D7D9DD" + fill="currentColor" fillOpacity="0.4" /> </svg> @@ -155,7 +160,7 @@ function MachineIconMedium1x({ className }: { className?: string }) { /> <path d="M16.1875 3C16.6362 3 17 3.36377 17 3.8125V6.1875C17 6.63623 16.6362 7 16.1875 7H3.8125C3.36377 7 3 6.63623 3 6.1875V3.8125C3 3.36377 3.36377 3 3.8125 3H16.1875ZM5 4C4.44772 4 4 4.44772 4 5C4 5.55228 4.44772 6 5 6C5.55228 6 6 5.55228 6 5C6 4.44772 5.55228 4 5 4Z" - fill="#D7D9DD" + fill="currentColor" fillOpacity="0.4" /> </svg> @@ -175,7 +180,7 @@ function MachineIconMedium2x({ className }: { className?: string }) { /> <path d="M16.1875 3C16.6362 3 17 3.36377 17 3.8125V6.1875C17 6.63623 16.6362 7 16.1875 7H3.8125C3.36377 7 3 6.63623 3 6.1875V3.8125C3 3.36377 3.36377 3 3.8125 3H16.1875ZM5 4C4.44772 4 4 4.44772 4 5C4 5.55228 4.44772 6 5 6C5.55228 6 6 5.55228 6 5C6 4.44772 5.55228 4 5 4Z" - fill="#D7D9DD" + fill="currentColor" fillOpacity="0.4" /> </svg> diff --git a/apps/webapp/app/assets/images/blurred-dashboard-background-menu-bottom-light.jpg b/apps/webapp/app/assets/images/blurred-dashboard-background-menu-bottom-light.jpg new file mode 100644 index 0000000000000000000000000000000000000000..59514db9d7a547be9c691725cd8076f9cce8c3d6 GIT binary patch literal 790 zcmex=<NpH&0WUXCHwH#VMg|WcWcYuZ!I^=Jor9Bujf<0mlZTs&hhIvNpO24UUQ9w* zN>x!^O+`^fSyRv2NK?l`M_I+#&BVgi-pScX!_dRm!@<|u(a9cU2qQNS4?i!zoS>kb zy_SlWJ;~tzAqGJX1`UQM%#4BzOoEKef{g!<Fo=Vk%E*8W7@63B5~AqB3=E9_Z!z#N zGcqs<Fbe>AuNoN`L>eGEmnwJ!YiTHGuxPL{v?_WqItptlhdpQwVu+ZS>6BR##K7ew zvQ$xkXALW>#|8#o5fQK=sI3}`9t^DZKszM_fKFjXc8ef`LO=r0S`MHDBP-K$jsqPX zz41V$B7&%nW?(1)D`RG4v}Y8=qksvnKmfZ5j0|k{Odu1afvy3$2cb-mfl)xgz`+sb zAbZ6>zl-}NXM(`@)V()nzBV>Ct1eP*cTzsXbH4anJuJY$o)U1*-6oy8?KTiO^j52G zI<3+&D|6fAyY;bOKlx^EyPV3j_|2qD)lIiDmqqz*yO}%fcJ8Lg+->qer$`C{O+X14 zMnMM#fdIId1@_)3(T>;+Qj05OKpqzW`Abd!Y#0-gd5nSr3W^4X4vvLzuL;Ck<Z*Jh zWO3B>wN9Ad_GY!~i?o0_i#BbVm|SR?%BOoztd`x$_(l1|^t~=EtPh$_9h$P-&?|Jg W;h{x~7Pq>(y1H|9b#>MMzX<?e5V=VJ literal 0 HcmV?d00001 diff --git a/apps/webapp/app/assets/images/blurred-dashboard-background-menu-top-light.jpg b/apps/webapp/app/assets/images/blurred-dashboard-background-menu-top-light.jpg new file mode 100644 index 0000000000000000000000000000000000000000..08a4eab1d167a60bc88f7d6b025fbf399de0aef7 GIT binary patch literal 3588 zcmb7GYg7~07Ctj1m;{iCnX2HU2@WxF5rmYY%UTHmH3OuCsJHS^0fT}hR;;=d`#=mz zK`Trm&nl4y?6u0pNF&x-t%{1N7J_vlw7xA;A609;UaNBV1Z{8ckGs}Avu5SYnZ3XL z?eFZfCj;FBmk|*+gPwsf3_}{Y&_FMWL4qMXKF?dg=ka}g1it=(LjR#d{a+nEQWO|0 z884+I)Hr$Q8%lY|<dAXHYcnDyzZs>DQOm+LT8&EkMzlJL1;KoLef|CX#|njGqb5)j zqW<qOa1;r7xD;LY!h{GHVqQXQ;5Zt|QU$XAF)-ZA8{WL~U&~;D4fG&iFAU*?7XeSd zNsx^76YvBogej{x|LuH;SdOXW8iBeDVVWw7^y7*f%d36#!X<u9M>vQi@Ny-iFl8Pk zaT2#8b%9Dxa*$1><LOC+I5J5z;sQdnfTu%<u;Bzj5@Z8}kf;#R$q-17*C22MQ&fQT zWPz=KEMh4%Mgs+e^aV&P=jjUx(hLBzZ8oIh#Y~r%u_nldct{dAE0_%<K<teG9#D$J zT!}`fE`y0oQY*>a&4uJtVwqSj5E%Lvh4fEQ+HxXo9uSgjFMTbwG*nj)MA(>ixvp6_ zp%}t>23M@q0}V(3k`o|~#Ol7G6qr^ZR0J2o5(@MLNfI`XF$dCsNRS+k7w`<Hb277* z%n?Smym+h3ifCRZOR6s{Bq3*Y0S}x6pcPy|-K-4(=QvD&K_d9*Z?_E#K#HI_7i@#t zHEIkUVuV|5AS0_b(j$+`Y5+rWCLlshqHH!$5r>O5MjU;}A1)?)ZW3S*s@NJeF)G3? zlSw@6AZuPWa73UCPoOTW4y%sbt0b)m>zockf^>L*#HL0%6rsbw2(JcUIcp891w7s? zaiEM%-?PQO)!N-*>vQW1JXAmevm{QXV+(*aiJ1}2hD2;uKpls?VZS{R6?#yi+Q1TI zFcfq3f_f`Xf-MhriU28PtfB~!X4avA-2}wp$}QeQ&0>Dsm!7vO9g?h<f6=JNx318^ z0}Oc?rh%Q5K|qAKN<HKcsDoia1dg1oa2;E8I0T8-=vZ=)j52mt(3}QX1%W+|um~vZ zfWwnuMLN$`71;0qWo=qx9MA5)ZS%b5h~+?xm6yYkVr#jGgX~a;Y})NscQG6c8B$^O zRw#b+(K{RrD)ihsn)GX|0O6{|o-h!WEeCZVP=pES0Qoc~jGQ7=+8gG9D^MdyoM^YH z1bUB>Djhg`DwnN!P@JnYeAHl;L3(h|aJ<acm>|{e4pN6Q<(YU%Z3>(VI~TUd#siJG zTsXu?V1N}gSY|jBg(rW)R>aEpA#{$h1cop!f=2Oj2oXkzRl2bBf@RP`c+hV!E`EKg zAh&Mx!L46R^5uOn_r%#wpD{C+A8d0G<|%LaF5ENk+P?XGp8EQ<h^ElCJjuQAp|w{Q zuKIT2p~Tpx`^(2PP7dF<W5K%+juieGXfUA76T}O{qOkm^|6zguX#te~ItDQ?<Q;{3 zFvbv&5azjG8e&31B@WZ2N59M|N^<A9pEWTjPJh*!c<Jd_#RvXnogAl#FiZH4cj>JS z^66)IYQo(18<<p1jF7;?_LIMu$2;GDbU#jS{p!{Ry*2GjMvy4>Kwnm9q`q?7t(2ng zd2f~OZ?MMXrzI^{@^b0o;1iBjlS){)_*3dvYetCXYO1zNrmbI{VEh`Jwmg8R=A~`U z2!WvOPXm^N83fw|$S{$HdKXiPgxT%Wmu74b^{w8JkakCu5f$X9%vp13Yo8e?iMEo5 zHYNGCHi&Mh`F<iz@R=wDkUQjmM_<fhJ(wEvAZ?$7ryX`JZBoxg5iD81)L(tfd8Q>| zV%eNLC_Y{MW_DtLJHI~`04QZ8aUkWmv@E{gnH28mkDC`d5dxeR=?--io*H!fVBE`Z z4Dte<iA&F%J*LPDii%#--93tmc2i`H$$4zg#ibt}j-#C&ijBJtIpTv`O0uQLC@o{6 zx@fX6J8YCOa+KVDln$z)m8roFw}!INs4C0pGEhtorO%;yjbYu|7Pp3jR29hsh`s(! zSu%ozi)v_UUy7zNHQn1~icHC|Gg^n+*Lyu}x6pQmZmBgoTt-KGmcy0t${}M#uaN<6 zDW;RImCjz1v)9<-p67O%oT(;f^5n{Ris@B2T?%KH!r5*rIFjz&jY$_oJujmpEHsDl zZBA8489v3(RJ8#>C^t>@(8K=qQ)P;V!+=57Bdy(Op{2mNUE%Bo11%WuPgOX3XnT!@ zDmU~#_iwS#K}-%FuV607rHmGA%6e51ZrToL0hH5~11=G@^a$6M0p!k+GA1cuV3M=! zES6T>OF3z(xjZ_uCi>uyO}+2ziKxkOSPV=mRiUSyZVXg#jxtdt*;G$Lg-ZeyLEnh# zh{Gu_^o>l0oCozaav<cOz88QW?a3#aahWz)SQ0Je?>sSOQEpA6B?qU29Vx7!6vJWK zO)tddXE}e#@n&eLn;yO<>Z02OUX;$z+-quNXnP?o?qat=T1G1wdP7-;y$1|(sB(&F z*HRF!kPiBBqT{8w4i|XwS%lgUN2-Eyv*JefJNtSx99CRIMRM>9aeIx;r%a8<{OpXz z5+70-pH<-kai?BdbNko+hV3A3dvHAsQ2;~>(gBnm(D+o}T3BVpRVFJcm_coguusk| zkG9DTmA@O9PS93WoBCA`Y!$1mk%?D0<2~ArlH(S>N84Bqqoq1>BFeHkb<nT^g>0kw z%il6gs3MmA+1Q91KWzWLDNmAXzCW2gIa9Y9>!P5HJ-#^3o}E7i{NGa%1%tk9%Ofrn zElcke{^~IghQ5X{4i4S=Wy6NPCYFS&!lE<s)+Lp9-rMm<e<R{ECoVcZzc@BFRvjcH zieYqj7gw>r#pdBfhsgYZQ;z;?2L{lTl*{jtQzlFpH*Cf1OWc!1o95qdZauen<8x!k zyW3`dy!c^S=D4Mm=eNvVoi<|92=Y*3Oh)#QvZMWXU$;+wr7$J_Z0pI#okM=Qe4ydG z-AR5oeS)q`zkJxeseWzgru@@^vicHPg03pgC)JhV<5NgzpwVdYjMXVba+*)8+4W$7 z<)1U}hDPN*nf35ls65eK*C1<c8&|mF_V9^i&0nsHjQp<Ox$5+>wL?2<Haq>wGc(#c z)qTDPVmCIgDI0#^a+UhHl%FXCU8l7_N_2>B*BO2ve_`6Gm(!<h=I~Pq1$#dib@Ho= z)!7@4pT0(#oug;W7}qg?<|n39dZ}lL0*Lrs8}28x&bc>xw*FzqQQz=%M|p0SnkQ{r zmaK90?L3mbE{G^L2aQkc6mmxM^O)tfI?JuZo$KG&ywx){^&k86X6t?;=F=ygk?Hoh z6>pF1x%%jBPFrjDk-f(D#Ky-mx~q7LK(^!6>RYqBKif}80>shsu4r4y!gZ3sA-95j zY~O0Timw@xr}IlhWY{}XrgOjQ9^;Q6X~`@fzb)@`qk82CueBMg#33u5ulc$qzw>6^ zoA;BNE%l|JnP+vB=qmZcwq>1=OwB#pdiS1ur2cxt&-o*F@7~>z#lNz0tT|a0z2foE zu!O_vtO*l-ell|Wr{`V^`A0dK9a?i_f9>=kA92z{YcA)KJJ>JXg1cXwT$$JO+r_Z~ zy88DQsF%)AE5n0$X$zmNd2>mc=t->lJ;Kt|I(qbp(U~pzO?5lY+^iiwGq$RKZta-T zkFDKHioZF(_#@tm$C=iKw#WB=kPo2k^wLQ0;dN(Y=a)R?>5`wgA0MwNbqZwmr0JxT zFiK^6Cx0^YisT9{@^5UGjLcuOZld7wvf#iis`1}HoLjq7ctM@H_tKo3w*5Zox40Yd zU-Qqkp6FS1J9kZU>|EQ^(itI|oi`qeE%mp<b~IAQt1N*ZeiI(M=<30rLWihj>en;U WKbh@wy7j^DH-q;tKEGw)!oLA@?zqDM literal 0 HcmV?d00001 diff --git a/apps/webapp/app/assets/images/blurred-dashboard-background-table-light.jpg b/apps/webapp/app/assets/images/blurred-dashboard-background-table-light.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1c8bb8881e73693dc53056de585bda0ecc018ff9 GIT binary patch literal 18021 zcmbt)3qTar_y5eY*esCFE(nURSzMm-P$2TFD6=af2qbK25oW#sHB2Zg#MFrvUGcT9 zh++y0fsgcCiiW0^N{FDCC}|>Tpp{zo6?|nr@_+7~o!wcWe%AjOcg{WM+;i_e_jAv^ zv(uCFPg)3iY*b7XL6Rg94gZKI&BQ3e;TcLsb#stWGABm|rylO|9^Je57~tB=+1*p= z?WIzx2KojL5BBvR>OW8w5*0dhM1*FP#%D-$d^8(BT&;;fPDn>5rykGt=r5P|kML9Z zMf_uWQbouqyEmi<?MOKxk&||E@<}z(3k?+pD<V>2*9|nT9pwZ;N}gOGoa{(KLfg?$ zy~_{|tOJ|CI#3QQVb9nzL>W<LFC|P|89`{_-&3n6n{s`}YB)k_uZ#wdR08_@mvwF5 zdZPQ%&&zxrs05O*CkRUG!kOrQTUS!-OWVhi4g?`_A=m^QDcVF3N>&QKf&On5#ka!N zL@xip<<lroXqC|<goS{>LP-BjRbxQ?iHCvve~N-q$XSWOZuC=_b~2%n<X?6AZyI&# z{vQc+8cscd>(GRX%zd@fP?WR_IhG)3z29L6K!r|YDD%<#mP6AeAxwGjJMlgpyh!@i ze^@$mMh7Wz{H+ASeV7YDc28xA3$tescD^Sk6Xnr6PdAtVUe11gWH~T4B#|rKVo+B$ zM5TBbsL#Xp5TsLC0&!-%iEvuY_>yV|+$nIZjX6{#60W0&0BnJRz&i$%d?VV0Kp>S! z3@#wE*2?UaY6paT1ZTbn5?Y9A4rtp!H_B*5&lyS<%xnlcLWH~uBozgY1|b^b(&Ocu zJSz@AE4IV-8oCTU%Y0!eEVMWt+i6@(Ad(;-i1Wxi<DI0fDwzutu_4mQV4{~!Fi3q& z?xMaWy+e^DNv*|e#&Hl6P*ag>xB7IB8^gO4wI?*t0woAka_}c=#H)~kAkcg@oFEaN zf(AVqU&4p+k@`sCiRxXDxS5TbmKg9FQmM7G{5S$Fu9#48fFvA#;DXO*JTxd+TO|vf zI&BGv+mupK?Ndf2mYoq<Lle5rfg*e%Ep-{a{7;Zq9sJ?wO|9R=m~zYTiXerVNQ#g` zUXyTIP7uMo?GaDo!BG^9G6$4kgQSQM%rL>p1s*YSd4WX(RRml#O&${*Z-7b*&}dUG zk;_lptkrO42AeUya2NqcDSFQ2_k&dnqv9)KXo`Jtphh|~g!8wvP!W@qVF(EVABL-0 zLuv96r}2ChJ-~uM$wuNG5DcsEqv<q@#3GUgYjUKyud;M0H^XF=Q<1nksS<AwB`B~P zx#uhgLS)7_VEKm_A?SqTBM5^)tO*re5=6)n(2fS090DEV1i^}(ceLuV&Vm$TFX>(3 zo@df<RD!JsDZvjBwY5hD5Tqv{3L7$TKKr0i2XFj5_??0iw2Q=r<kJH(!Zb={FO41# zd4!CHrE)1Zi-jzroQnjO*!9}&9lad}WRTz$G8@BE5fncH+9j6|eAo<y;iKPm5R_^J z6B0JMFb#`m#IW#(Aowqczg;Nl@nt@uV3{3p_$lrZWpkihT(VH2=zIGW%q=)UYDV|; zYrqOiL31>^jQF6<Es!O1;>}F4AV=KN6LDLT(sy|@&>xQ$O!+72#Gy3HbVwm|IZ{Jv z=HOv*ql<YJ@_;o9ARB&%!=t0PWB!H9n|Xq`l{Fv;E><YPO7USpR&WJ1NOF*8J{9pT zEv3fbf>HnmokeJ2NZ?S;;PbCI(Ab1Vw@JW_c-(yUFehNxR8k*^Ew%9-gkOnU6o(sH z3)uP`<Ox>dYP<EP<0K&sevt)R2@M5GFv?}hecK6NBomXAfo(&V+1uo0bd&>eCMYC$ zj=;epLA4`+DSyy)RG_*P{yNRPE5pB%0eDRuY^x5$@pVz`{sz1ROU@i?>wtz+$$`~y zBrA%C(gE$WPN)9HJ;x#Z`M!?Yo>6%n+`##V&`Ub_5TeAaTr(Qk3Gh|GB3kYbAU7eU zg%C_IiD0DRkc1B_V)=QB!wld-g!3*Gxgwp2C2|>G!X6nxdcn516%>3AeC;n4=yeYD zvp9Wm=~iTG-5F3JQSY#E;3bPiN=yu1Z8L%?03I6p@wdLMJD2RwYKg>XZjUHX;Ka#5 zP-|zB^mrmcbjK6&h*LgJI@UWx?VEmwp~Gl&m|4kuhX9<2Q!|T}lVD)>fMkv!@MZ!y zc0v=J7%Y0f(sl+KY6~j~f1>5VGE2chkkB2Xvs>N-+;S#hb?UJ@%i#1VgNr$=hQ*-; z0LaKpVgLx8FB#+sH{oiy%)4-KV9|THZ7wefcM7QzP+~xvtz7T{J&xUjD(0&MSeUO8 zekaj|E>9i3hk*ARLOQn8h2F;7g3TxfwV-lgwed9=xqvM6m-mTRS!{rc1T%-WJtjdQ z)cF$$t4$!oap{zMVe-mo)KZj*VGU3NAc>1a0+$Nl*tIMOtw!bwM`$H*t%twPI)05< zXbcYcn}x8uK*p0)3RiB1Fy#^^K?Kw1F^&f;ftMk3B>TbD_}ee+8Pc&#SP63*xEiwv z*ov)PUyh$eb`%Q*Iv^MOFTT7(HUWDDk%eduesO@kD?Ve*zd_jI8f+|$tV}MTESKMs zR-uK%!0W0RvfwXK7F%0}bRbE*Lg-!$uK@7VkBtF3F@VHdhEFnl+Jd|QyAk@>_({^4 zWPi;Wg8tl*uz8UqG&C}Du-WH2oUElD<4vHFAOrk%_tuUkoLwk@bAYWEz%n45`ildy zgCnJU0*_^z$p~0ReVWM%IdmujnNaah$?Qd9n^m923yL4YZ?B3zc_yA0HIp)^BH_fz zn!`(Bz8J$|aV8UbErzuGTC6dL@o}+9%COl4^yYSr-rNS%hYtSuS35|{RXNfCrJ6-@ z3nBQfWnt09?{-ogUg3y<n>WhKX8f9uHQoWbz=LtFP_wv2Kg40e5D+R>wVAUy*E(t8 z$V4Y46$x*4(ONN7ktvU07?_pB2o|=E8Nv39=9<ZBi8wwXOLFqXVl#mGcQPj0WCLLP z=v{uDyx<$2-{G~a%>vmbV}^nF!Fec=_nd4ZaCAkT^7s`U(O@ish9wsZO;At-g7Z{# zYZhzxphF(OzX5gqVk3Cz@+n{#z1{6Z(YIXz79wndgbpybPzMk*gvn%iyCc=Ki&nb5 zl(Sd^GfG8BJQLs!vOy%Jgxm296G*LDWNOL2?G6A#Msy<t11SU7W0)<z8wzeZzTBao z0lSO=q)GT09JTM7Q1m-27=QHh<~9iKq~H%p0=?gXxeZnU*FuLTfia(;wnwnXTieoN z(svR>wC%kNQh*L!k)d&d#u`4`D>7K{`+wv1uy@w_`?(OB6}B@a;q{L2g~jDeWxggb z$0XeDDBHJ0-RLA1^d@X06nLlMiWmxlmf(PsK*8e_AryhzQj@p<hM*26opfeP$iJ)> zxiG)t*~UBvZd&MPOaq=*rzJp%3;rNc;Uj!&r(8~wIED`5<&p5HX7Gw}&@)PamY>-+ z=YdBt!4<iTfFCL4KChCcoK+*dWz@wZ8O1_jzDl&1Y|x6-B>c(-Y5#9BF-fXP76yV2 zOVq-aC|DPc52mdG*Z3fGp%6sk)FH_DmgraFT*T_s6ATD&zWmKXurgmI#M`7}A|_cF zHGh>L$(isfTShvH10@YMJF{0>#lmkq{LgGku)rzQ8{Lg#9ahJ{yM3;7S;BsZh=t%b zfnK+{GJ6wUF7joY)*0A0xh8mhqZ1$-Lk*43LKfb3qnPzlcy`DKmX?;6J*z~_!CMi0 z?-EPIJVq`q#8QFhOn5Wc!DqkzG$Da6e0J7vg7CSJ&M*v4;8+uU{-Z!a+DmFSW?mWP zk~rG!w(E$kw^vn%=lN`#Z|}A<aWn`oLpjM4n4X_~?v(XK$<g1JZRl08Y}wD@6B{ny zUgZ`#B*o3G9x1N&51cXecwo@_mZK}2URz*je|wZ`V&?nDy7|w@oOevr>fzCoPD!JW z-tOCLm$lUfab@G=hRZi$0xoiE%HZeMf`<(=iXbH=e`6uBTF_nlAc<}f66|9D&3`wA zSeZ@b1W6|_gV_vjvRa5!gz|ClBfE&Q${!l-F83}zUw^l-xbZ&^!dvBi2R`_tu&{VU z;ho;_C@(B5e2@f1;e*PikHVV$Z@+sf^^e~wQ&VRr-EK}zO}n0cs(<REq`Pm<t<5aT zYz#STH?rVrZSUf!tM!th^?wdeyHk0gBDKE$j~n&v$jzM}9Mh9F$zA6i7@%(Vh&V3) zrSH(lF^BbkF1~o>`Q@jQZYP1!m5}cq6|@&8hp%|#e<!5sQt#Drpr25IsK4w{^{MN# zKc9ZGuviVoh-yxekDS`xJz99W{#1199r^cdJr3@!8hCb~E^-XDazMj@uf0;U>R}vC zP}J9->W!KmYyV+P)+y7Yw`+c=KM~Pq<fFFA>GgU{?X~<<zY=%(sLk9VjP6!b5$<xt zsawOR*0-$e>*0Uvy(K+{LjU@^FRYe-ty?terc)D&VPR27O={ZL3;JCd8FzO;2;V`g z{OW4#R4yOTkp5>u^Ms64p!YYU5-4r?6Jk}fYg%Mv<d~4!^A9M!{(9S~_NzOFMUIKq z>sMVHaVZ2%<y7y&_M~6KEln<k+)P_~Xj$>5;)Pks!Ku(3roO5d%!8jp|Cx_AqgkYN zLrqdsv%mo?ac$ai6p{M+_MaENkycXa{|#yZBkb!jG%_;je)9mkvW%5|2bLGTnH9O< zSbhfb_So@YNNU<=y>EesqTM0KR+;W6*DOfC^?cjzBc~392NZ@39Yg6?`G6}-yC2Ak z&os}fTZ1DP5_0)VG=j9x{s`EWn)!tA0D_gunqM_^ZfoO};ve;Q$9*6lAc-j1n_hS@ zZFzg)tx52xKlR7#D<cE`NL&5Yy2rb3o^ni$t3~ctsc<Jy{P`gQ-0qJ<LPk#GyBax= zpHr(mZT;Nqjjcsq*{bX<hxM)U$8wlM>O!aX;(joT!uuhAzMOUHi=w~>^RC@=`Ua*y zP?z;1iXu?9_pOg2$}7))^u}_C)F~s17f^d)QN;SkzZG>qzk3*(L%4C|nC|icb}dPN zhOU|n<5?-MiZJb6oINKA!}J)ydgGjr59JnhU+_ok9H&~n{<bjmRoB86#m?RyR)+=y z^ijB1+78@H{i)Z3!u#z?of{dMfvk6jdkkCxK$G_yxFkPo`h3-x+SbR<2>%BJVF$^D zZk}DxlwQyjP|%!SdE%e&Ndc7`qUn#~!}`cVKZ@S@)O={NXVHe4p~l&JxR}rrGrgXk zZ%RL>ouXPDFa0P+Wf;voHBWp<xEP^HjLH}zU4zP}<QcFxMl(j4)EGnjBVIu;*88h) zMm^1Q)sF=+%+nZlv{8Tnt~mdkxPRW{|Fgx)nD0A!pdNJ^#=u+AaPf#xs0l)Lb>+DG z(8R&#<Bko<vbK(*a3c%Oq(*Oo=%&iaph3UF`+P?$*`=0_BD)_Rw-BUz3^b&k5R0A= zkHA8rW}vhsm2DN(03ZaRGQyB$fG-A#NFD+zv@!xuM>FVPMhj&l$2LTc3_LTd4l|&( zC4^bCTeAY)v&=*i3IYu&>ujYU5}u@M^6bhJDF&eqPcIG%axjv}3SvF?z%yf$_5BFR z5xFJ>7Mx7k$yadcg-UWLDk&HaYPj-?E<!;A_R&Nv#1EB0b(tz5Mx<hq%uFo2fAW^V z2ts(8d6j}yfq_D-5?KjmViD#^l>#qEbp~OO*=k`S5DA^)3WrKOLdsR<gl@R^zCYLe z+hHWOl54DIL7k^fRFRS#j!Kz1II%+y$~yL_h)FRvr=ZXY#Bjioe2-zxv-6?0hJ^wX z!U-l4G8Tg|hN~7i)PN@iY_M3%7XnXH1Q*?f9En&Jja92!8C12QE+EHQ3}l3nWuTUI z_K;P`Ye_|OO2NhH;^257jTt$WP1DizC$BBwkW@^ru_FuOZLm5JsYx1@GGpCeL5?J_ z1&$#y1(M>DK_%LcVind{3X;L^zm~d)AqgW!h=LUfpN18dqdG&{F(s4mzbl$aA}}0c zcmM>mlXXz}{o;kD{~ECz=`gw0Eh=c9UC}(-IMyuT_hVmFqaZj4IirSzB~5YVkVqgh zI5wpMC6DxqGlGmm4BsO_4lPHQbL3)F93eHx7%=Ea!W=uwpQ6rBytu@&wJmMXN_Cax z7~Ud>Kup%!iid-Ffl!i)<dlNC=_mpM@xj4%GGh`(&gA@i@r<B1KNT@S#yT6wiLfec zWXN(x*0v$Z7zl!6rM6P2F-Rhl7<`cmW=f;FnJhfCweX;`@)YDZv@QzP+(h({LXx)) zjz}pihc1KQL}J8}1V?%m;$ONg8V+%&BZ8uP+3Vn$c5*+9>SA(@2`q>y-zgG#u}~pH zkpzN+^VpFgn^9Fvq#|Z|L6RrJm4!IONl`&n@{n82qFP6aCb*6wp)Sl(sQ=?<u}PKF zu@MhBmBA{<gXGr{GC&Linymc2PsCp(v9T+pxun?@!fpha2s%_jQvjO(<h8$_D#Pa1 z!ZkL~7_*>2bR!X;tY>0y;1K4}l|n=aa}3mtR+(t5GA0K@#|%NU4YD`Koh3w*=jB*Q zsT9%3kHQAcyk~5{FB{L=@aRke@zD$fG8;l^4m^0k1`vqh5Qp7TVa@3}sQmo>pR;cq zxPasqa%~PQc)h%$)*>MI;DjN8up!k$@-M^%AGX@S0D{rU>R%xC#vuct$JwaIHC@3A zW?_hCurX?2Lw}1oXb2}OXVbt^k>=pDzqnm$R7(}4iomf#YUC6B_ZOIx=1|D$mHgk| zNUw-Z-+dx&FNXv`uFca6Vp1xi`5G$WstOMAolI760-vli@_)an+WV{Ly5@jl4yl1q zn!7$*3^{KO#N=S46r#Ga1dU1|PQfIDE^~xb1Ub~gj4@K07o&oTgoQ#1e*z(%CV0Gc zlp`Aou3`*4ojK5240v7zZ(Br6XF0ORp+n43I9WSFuAANDu)+X~H?U$jD0u%=WgmRb zy;oh(9NL1!R&q@aEO>3YXf@0m*#()%3S#g^Ru37vY=*TB;>h5b>BeO2)}r7Lr$PmQ z979gQqFM*yVwfD!D%xmCv<fy7QAIpTkr=b^8bBdtMG0YyR&j{65b|kzc<_N@kcG_L zW7gVR&Ee;oSR@n3HC9z=d$NkMoPeBJ`IqK+uB&5(M(C-Ci7?jvv{NJ!WjUEK79YtQ zOYp!%Mg++i2`MKNM}QwHHj**4TC5Vc{CnQ)#K``=S!b;|Zr^2r2gXKCdNt16+{lQA zX9Kyg!vQ+5fA6gE36XBHX-Qzr{n$A=;mAw~>yaVEC5o|WNGh7%G{y#~L_#TtDp*2= zorp(W%2^~5C12pBgpjIi<;arXS7JGq3I!`LT|N0rS#b%I(@~3;)~2l#n=rg#4qnKt zO%<=(*uY@$$?C&f@udfP@b3ZCnL4~7XXUTEC^kX7YfR9&>x@{+=f}ZF1n;c4;YK)F z#U{Y>C#%hd<cEswNJ!;4yh1BZU=Sc+%dzIVNKRm0%D>Sn1Ov0(|NSsyE;o|_Kk%QG zNWwh^4p+u1F=GyokP7=6C!MPojX&I2(;zZJheu##jK4u7<2NJ@jsUsLAw=)V*p<bO z3_d-*qHd<iB7l_2p^6k0LYi}CQO%YhZVU(U>5@>IV{A0yg#NcYkF-rFci1?9U4YY^ z%E(&ec#wR!NR@#^Q_I?9OkQBJSYQT04&JWn(k)^ha{S3^jKL@CtW6t|pX<<25#xz| zIZd)Q<fBR>M{s!fqg&v{ZJmZnO2I-x5?vnyGPtOsp-Rv>$^e6m%{+r$gSLa5w^13Q zQ9}g_g%z2YR~aeBsAe#a3Nx|z$u9>>yqtyn*bv!GG1JR9St=4T9I9Z60D!A!&oziF zFpIO?PX>{gkE+dOuJJ!;z&1n%H%&J-|7DhQJv;+-z2>k@BfbKV;NRN2&~=8Ktr0}H zYl3CMFb2rMkI4KfD_9DUixvcYSOm!BL^Ghs%3m{Y?xmO!78_u~npG7ss)A&Hi%@hx zQpRR{K+f7!^jp}5Q#K8QV}c5v5bqg9Ej>7JLJmfrsDc>PMtE+5GzSN}by#2kKRGh! zEmAB*Z7`84Nr%Hm(7+R6f)v$lScYI38-b|uw+&(8(L%xp%|3GUy%xyQl$2)}Xu`D9 z^5v6$w7BT~p<lEt+c8D{f!ixzWqJHMGAd|a?Xn${=e1q&@ISaq7uD_9jw!wWcll~g zAgUncXKR<Zh09ZZT0U^ZqAvR(pd*ep@cp3gr}U%AoBehA@bQkvmTB*Wd->;HIl3lj zdR&r>&ice{m23Hpthgk{V|gXP_acuPRzH(h;*#}w(6p+lwUfe=f*(!NME3WCCfnxs z^Wa5-Syub3Nyl_Y4F|I-Qtlr_Jy0JXy*W*Z&Bn{tRe0fUvCzZWRnvaCZr!fdA=5p2 z^aTEQ#0K^E)17DnozO=gn3Eq3Uvfx0sYGHYeDo&ev~utehE*qIWKRAdams3LS6j&u z%Qsy*R4J!M!v~@j6#7iG;#q1bdICjiJUs9x$Eon04R|O=X*{5UNKWhZI`on26|ii; z_6-gC^_T&Lhldh1bp^{%Y#$ox;e{DcP~aP=W2cK-e$q!#pENb5-7P-%-SYSqVT+O; z&(5ko-&v4i|Lr?HH!MBqHh?<PP`zfk{n@Jx^X~Lt@JzA#>Kjjp%vI+WoeB8i+|^(A zJbrWGfpHe*yK%qn|0v|scb!w;`KBuUn=j7C7k)R~L2FQtRrg|xk9_9z*&sLB?j=Wd z>gXJb(b1j}KV5lSn$}G|?%b{#pDhdC&RE^++Ao#8jvtjO{La5QH|gjz1zVG2<H}w+ zvi_Tm3+6~_&L8>Y?)OpeEY0&Nt*OlTwPkqe)aomlgHJA7xwnrt@A#lzFFzA?zP@$0 z!>YdBw6YOz^m7{6W8&F<Q~uL?jbhn<KY4k}X>R|uaDA_&^vKsYRAtNB`{^^r-15FT zr)AHfl!rl=<}@ul@q2E?1$k=8^zbt0yPuh61b){$c<E!uLn+Reri`rm`DcA{c(=qo z@!j%{Nber(R;{FJ^woJvs=82<u5epe>v^!juy@jNw?6*wdMc`wBi^TTJSZkd4m0-f z@NOAg99eyT!@AbgNKI}P<(rYS^g$j3NIHFaAgzwQmY@9XxOr_8UwZvUdBmCGUFmJp z_7C4eRvh}V^y?I#kN!7lxN}NXc7D0D@3yhprR&roOBS@Gt<Kzk?5p`c$pNme718sl zHqT$&8@FY*M-6FM0H1LswUctBdrbw8ltU;Dojj6?U-0eg`f*Dq&ZEbTo0*xJnECSP zr0s3)6DLl*xY=Wt>!{pKL$beX`0T~6n$Cs#Eb0EZrRZ*-_^U_a$4;DBlHS&D%;`|~ zMyf67)f&A<A2v`iP)XG$tj`$(h6T4}Ti(pAX^iqzD5)2|KDLNr)<@B5xs=vW`U3;{ zSG_ZCsqg5v?bD_1rGs-EqT~&V>{(i>uGbiLIkQh4J+O=^lYRRA-mS$`#%k9+^v$Tu zTKQ}5M(+M|gP!wE((L<o$CT_zb}4~_`lL9g1eW_|CPlW7J+93!otN=j|38+uejiY< z%)M;2bWP*+=z%@2udzSf!~OQPaoe`7y7$_sSLoMvC@$qBKPs+peyAMM-TrgRb0k%) zIK9;Q{8#%wJ#ad6!Q+QPO_>Xizh3b;|3F;ixBmOxk6c|Hp?%Qv$c>K;rD@$C=O>?B zdUNE8?1ah}qk>nIMuy8Ay6yayDcZuFj`aC3a%!Az`ulWrbj+Gs!-0E)3+_0-aM3Yk zrLOcvTBoGOy-X=7<^xKvl+t?Losfm^4qmTQ_PBjH$bPiT{qg(QDE)GS_pcwTrrZmR zeSC+iTASU=^{(5Lu`{&mWWQxh+aI!pT$>+%`O9g228E|q=wRY;r784*-&gke?1PQZ zzj803@>Ty?#>j@{OS7AHZhGfrl4FB9;F{+lSauzgi>52x6mmIgC^PQ)^JgxXmR`Q2 zUE6aZITo<y8@ufD($&4A8nS~QKbR+dJgQ}0-(l_%e|)C>cvu#5peAunm{-=t_or#s zg_gSY@io=d016sP_rIX`Dk?j#Q+Cs<y!y<QDjgKl6=~hu+thPHJ)6~zsn7N@#G<)t zvbU-!HQ>O5s)Ehn;W7W3UaxU>zC29($gfH6{_HiibS}kg@n1T8T)*YCc6IJRlX^g% zRGwSqc=7&G*SpR?z!*1p1r#}<FtklT<aJ!TqGI<%w~t%Gw3qXD#J5(g%`YG1v%dP9 zephZdwH~~eu&*gaUqQ_nr!i*cOz~DKv%~hBe?ix@FiG~_&g2HqLnS34!(310&}+&U zE^s^g=6|LphHGzJT=w1k{Szm?^wLY((){f`=l`P3Ka>&})b#Fp{qg5U{W33I{e9vW z=lYKbD^9;P=>v}oMb18Pt(U_4#GecEKG?o3bK6_FHOHsURw(})lHodD<J`?Bj;W#- zeJyqW&fak7hWG5l=}~pt2G7Wus-Gaw*W@Xf^{of4EHK2*`!up~-i?ykp;HqB&|cMf zO_5f6C}n!hROfv<52cjJq16)DPjYJ9{2@a^Myj@!-XCvpEnTK^*{7wP@4BhvYi?+Z z(+v}YN)zX-xiRzY64*i^>;JcL%#p}Ty~iFunK8-K6uV)9mo|S#Txk~U+4l3jzn*s^ zi>`b%U~y!_MsLk>#a-tfo85=GU;8raQ?B(IQ*cLd>HWEJaWeryhg#px$WXz4`E<vb zI};~<fA;3}u@klV<ri8;Z}{-r@IkX4Hs&~<9_g4HH0rg1udMOOx!E-O;Q8V=b1ODH zn+_4#;b>Q1s&gOY;dna3-KW$gwAM=*sf<)K<V@mF;#`+DjNW%)Ms?{cI>@UK0q=4h zU^fRgj#{d8YoXV#UtD(Im_M#%d*ywvS^1-x9E19;F<al95}R8$`olXX$L1wwI|Td! zSS=6qjapP*)zorq?kAZ8y6ySXVbGvK&jmU*w)F{o-P5OVl5b?&&X3#{wzclB$+(2G z=W{V0RNYV?=7TX?*N;hfIqtiWA4sk=sOP8y?(1PJN|Z645RMz?DP#|4jc=SVr)6&W z(WAcaUXV}oIqSBiw>o=sc+WF$m|Uf9rCDLO7u6R1dZqM^R-f!M;(nh<{Tp%ghliK+ zc&R3jIkykZ9Br1N*<SIm>LmeJYmQyhG)*h<Q=pk=*XotjTXkrw1XU?zxyPzs`l9_> z_(JkZQziRqp0Y$rYxCj6@cxCmPBG;{_gKT$(mSDWcr1#8^$2u5)$KLEjn5p}wspy} z+%fTymj<25=o2*Br)cKbYpWz~rnIK7=Pd)QWmcS?@&*If0!`*d<xtP-X^E4bJ3Fj4 zw&BPew%~Ss^`9r74X!M)-*Wiv%bSn&k1SpAeCdPzdAk!66DQ32W$PE)FMbc3W8Q&@ zRAtx(`)h{|zvh(J)c(WB;lwnz5@p7?IW-SQR?V*u+qC%$O;dcYb2XQw4R8Ee(U8;Q znH$WP!$(bpxGvYYX!3lbKO9ZRYF)hN)&~5n+gs#4?x8MV%fu3&)pAOc>f!wPYh`ff z0U#hONZ}}r)V%v^L%{mDApQ0E5d*laJN-vIv%Hd>@P~VTxli$@Q`dQ3@3AFLYk$mC zdZ~0lZp5P?ZBxsM^2WtIuOvF2%jok>x$i55Nk8rYdf8#)@Za$^C`sS*ini86Rz=<Y zZ@Twu?=>t}G~yKcr7Rk4N7rwebw^&j{JiPP>Hz1HzS(o)gUa^p`|`c#+x%XP)BbPB zCpDMGG-UO6No)yOF!2d-#nsI;t@cn#fBU@3<I)q4m+BrCB#!vGrg2w!kjC}+j9GAk zJh#NIh>E+r#J|7a;V<oal`j1-(wWXVayUJ4%spj)nVvaNpbq$1fdD!f_C*5Z(l{6j q+$;J*Ku|;?BQ^B6YmiduWzLsgSgQMKSo{5FR;jP-ZQuQ*>Hh&`(97!p literal 0 HcmV?d00001 diff --git a/apps/webapp/app/components/AskAI.tsx b/apps/webapp/app/components/AskAI.tsx index 0d32265c251..d61ea0055fa 100644 --- a/apps/webapp/app/components/AskAI.tsx +++ b/apps/webapp/app/components/AskAI.tsx @@ -458,7 +458,7 @@ function ChatMessages({ <div className="flex flex-col"> <Callout variant="error" className="mb-4"> <Paragraph className="font-semibold text-error">Error generating answer:</Paragraph> - <Paragraph className="text-rose-300"> + <Paragraph className="text-rose-500 dark:text-rose-300"> {error} If the problem persists after retrying, please contact support. </Paragraph> </Callout> diff --git a/apps/webapp/app/components/BackgroundWrapper.tsx b/apps/webapp/app/components/BackgroundWrapper.tsx index c13983a12af..d21b631daea 100644 --- a/apps/webapp/app/components/BackgroundWrapper.tsx +++ b/apps/webapp/app/components/BackgroundWrapper.tsx @@ -1,42 +1,74 @@ import { type ReactNode } from "react"; import blurredDashboardBackgroundMenuTop from "~/assets/images/blurred-dashboard-background-menu-top.jpg"; +import blurredDashboardBackgroundMenuTopLight from "~/assets/images/blurred-dashboard-background-menu-top-light.jpg"; import blurredDashboardBackgroundMenuBottom from "~/assets/images/blurred-dashboard-background-menu-bottom.jpg"; +import blurredDashboardBackgroundMenuBottomLight from "~/assets/images/blurred-dashboard-background-menu-bottom-light.jpg"; import blurredDashboardBackgroundTable from "~/assets/images/blurred-dashboard-background-table.jpg"; +import blurredDashboardBackgroundTableLight from "~/assets/images/blurred-dashboard-background-table-light.jpg"; + +/* Blurred dashboard screenshots; the -light set is the same artwork with the + lightness inverted for the light theme. */ +const BACKDROPS = [ + { + images: { + menuTop: blurredDashboardBackgroundMenuTop, + menuBottom: blurredDashboardBackgroundMenuBottom, + table: blurredDashboardBackgroundTable, + }, + tableFill: "#101214", + /* `light` is a zero-specificity variant, so `light:lg:hidden` ties with + `lg:block` - the `!` guarantees the dark backdrop loses on Light. */ + className: "hidden lg:block light:lg:hidden!", + }, + { + images: { + menuTop: blurredDashboardBackgroundMenuTopLight, + menuBottom: blurredDashboardBackgroundMenuBottomLight, + table: blurredDashboardBackgroundTableLight, + }, + tableFill: "#f4f5f7", + className: "hidden light:lg:block", + }, +]; export function BackgroundWrapper({ children }: { children: ReactNode }) { return ( <div className="relative h-full w-full overflow-hidden bg-background-dimmed lg:bg-transparent"> - <div - className="absolute left-0 top-0 hidden w-[260px] bg-contain bg-top-left bg-no-repeat lg:block" - style={{ - backgroundImage: `url(${blurredDashboardBackgroundMenuTop})`, - aspectRatio: "auto", - height: "100vh", - backgroundSize: "260px auto", - }} - /> + {BACKDROPS.map(({ images, tableFill, className }) => ( + <div key={tableFill} className={className}> + <div + className="absolute left-0 top-0 w-[260px] bg-contain bg-top-left bg-no-repeat" + style={{ + backgroundImage: `url(${images.menuTop})`, + aspectRatio: "auto", + height: "100vh", + backgroundSize: "260px auto", + }} + /> - <div - className="absolute bottom-0 left-0 hidden w-[260px] bg-contain bg-bottom-left bg-no-repeat lg:block" - style={{ - backgroundImage: `url(${blurredDashboardBackgroundMenuBottom})`, - aspectRatio: "auto", - height: "100vh", - backgroundSize: "260px auto", - }} - /> + <div + className="absolute bottom-0 left-0 w-[260px] bg-contain bg-bottom-left bg-no-repeat" + style={{ + backgroundImage: `url(${images.menuBottom})`, + aspectRatio: "auto", + height: "100vh", + backgroundSize: "260px auto", + }} + /> - <div - className="absolute top-0 hidden bg-top-left bg-no-repeat lg:block" - style={{ - left: "260px", - backgroundImage: `url(${blurredDashboardBackgroundTable})`, - width: "100%", - height: "100vh", - backgroundSize: "1200px auto", - backgroundColor: "#101214", - }} - /> + <div + className="absolute top-0 bg-top-left bg-no-repeat" + style={{ + left: "260px", + backgroundImage: `url(${images.table})`, + width: "100%", + height: "100vh", + backgroundSize: "1200px auto", + backgroundColor: tableFill, + }} + /> + </div> + ))} <div className="relative z-10 h-full w-full">{children}</div> </div> diff --git a/apps/webapp/app/components/BlankStatePanels.tsx b/apps/webapp/app/components/BlankStatePanels.tsx index 5fde85cc050..53b552155be 100644 --- a/apps/webapp/app/components/BlankStatePanels.tsx +++ b/apps/webapp/app/components/BlankStatePanels.tsx @@ -12,6 +12,7 @@ import { } from "@heroicons/react/20/solid"; import { useLocation } from "react-use"; import { AIChatIcon } from "~/assets/icons/AIChatIcon"; +import { AIPenIcon } from "~/assets/icons/AIPenIcon"; import { BranchEnvironmentIconSmall } from "~/assets/icons/EnvironmentIcons"; import { WaitpointTokenIcon } from "~/assets/icons/WaitpointTokenIcon"; import openBulkActionsPanel from "~/assets/images/open-bulk-actions-panel.png"; @@ -742,7 +743,7 @@ export function PromptsNone() { return ( <InfoPanel title="Define your first prompt" - icon={AIChatIcon} + icon={AIPenIcon} iconClassName="text-aiPrompts" panelClassName="max-w-lg" accessory={ diff --git a/apps/webapp/app/components/BulkActionFilterSummary.tsx b/apps/webapp/app/components/BulkActionFilterSummary.tsx index 3c6871a23cd..53a20395f44 100644 --- a/apps/webapp/app/components/BulkActionFilterSummary.tsx +++ b/apps/webapp/app/components/BulkActionFilterSummary.tsx @@ -9,7 +9,7 @@ import { Paragraph } from "./primitives/Paragraph"; import simplur from "simplur"; import { appliedSummary, dateFromString, timeFilterRenderValues } from "./runs/v3/SharedFilters"; import { formatNumber } from "~/utils/numberFormatter"; -import { SpinnerWhite } from "./primitives/Spinner"; +import { Spinner } from "./primitives/Spinner"; import { ArrowPathIcon, CheckIcon, XCircleIcon } from "@heroicons/react/20/solid"; import { XCircleIcon as XCircleIconOutline } from "@heroicons/react/24/outline"; import assertNever from "assert-never"; @@ -306,5 +306,5 @@ export function EstimatedCount({ count }: { count?: number }) { return <>~{formatNumber(count)}</>; } - return <SpinnerWhite className="mx-0.5 -mt-0.5 inline size-3" />; + return <Spinner color="blue" className="mx-0.5 -mt-0.5 inline size-3" />; } diff --git a/apps/webapp/app/components/DevPresence.tsx b/apps/webapp/app/components/DevPresence.tsx index 27d96954758..4ce4a480cb8 100644 --- a/apps/webapp/app/components/DevPresence.tsx +++ b/apps/webapp/app/components/DevPresence.tsx @@ -15,6 +15,7 @@ import disconnectedImage from "../assets/images/cli-disconnected.png"; import { InlineCode } from "./code/InlineCode"; import { Button } from "./primitives/Buttons"; import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "./primitives/Dialog"; +import { cn } from "~/utils/cn"; import { Paragraph } from "./primitives/Paragraph"; import { TextLink } from "./primitives/TextLink"; import { PackageManagerProvider, TriggerDevStepV3 } from "./SetupCommands"; @@ -165,7 +166,10 @@ export function DevPresencePanel({ isConnected }: { isConnected: boolean | undef width={282} height={45} /> - <Paragraph variant="small" className={isConnected ? "text-success" : "text-error"}> + <Paragraph + variant="small" + className={cn("system-mono-label", isConnected ? "text-success" : "text-error")} + > {isConnected === undefined ? "Checking connection..." : isConnected diff --git a/apps/webapp/app/components/ErrorDisplay.tsx b/apps/webapp/app/components/ErrorDisplay.tsx index 374f427c504..4a1e2804807 100644 --- a/apps/webapp/app/components/ErrorDisplay.tsx +++ b/apps/webapp/app/components/ErrorDisplay.tsx @@ -60,10 +60,12 @@ type DisplayOptionsProps = { export function ErrorDisplay({ title, message, button }: DisplayOptionsProps) { return ( + // The backdrop stays dark in every theme (the rotating-logo animation is + // dark artwork), so the text pins to the dark-theme colors on light too. <div className="fixed inset-0 z-50 flex flex-col items-center justify-center overflow-y-auto bg-[#16181C]"> <div className="z-10 mt-[30vh] flex shrink-0 flex-col items-center gap-8"> - <Header1>{title}</Header1> - {message && <Paragraph>{message}</Paragraph>} + <Header1 className="light:text-charcoal-200">{title}</Header1> + {message && <Paragraph className="light:text-charcoal-400">{message}</Paragraph>} <LinkButton to={button ? button.to : "/"} shortcut={{ key: "enter" }} diff --git a/apps/webapp/app/components/FeatureBadges.tsx b/apps/webapp/app/components/FeatureBadges.tsx index 1364d0f76c7..706719dc9e7 100644 --- a/apps/webapp/app/components/FeatureBadges.tsx +++ b/apps/webapp/app/components/FeatureBadges.tsx @@ -12,7 +12,14 @@ export function AlphaBadge({ return ( <SimpleTooltip button={ - <Badge variant="extra-small" className={cn(inline ? "inline-grid" : "", className)}> + <Badge + variant="extra-small" + className={cn( + "system:border-transparent system:bg-blue-500 system:text-white", + inline ? "inline-grid" : "", + className + )} + > Alpha </Badge> } @@ -35,7 +42,14 @@ export function BetaBadge({ inline = false, className }: { inline?: boolean; cla return ( <SimpleTooltip button={ - <Badge variant="extra-small" className={cn(inline ? "inline-grid" : "", className)}> + <Badge + variant="extra-small" + className={cn( + "system:border-transparent system:bg-blue-500 system:text-white", + inline ? "inline-grid" : "", + className + )} + > Beta </Badge> } @@ -58,7 +72,11 @@ export function NewBadge({ inline = false, className }: { inline?: boolean; clas return ( <Badge variant="extra-small" - className={cn("text-success", inline ? "inline-grid" : "", className)} + className={cn( + "text-success system:border-transparent system:bg-success system:text-white", + inline ? "inline-grid" : "", + className + )} > New </Badge> diff --git a/apps/webapp/app/components/LoginPageLayout.tsx b/apps/webapp/app/components/LoginPageLayout.tsx index 3e698ef87ce..1db614eb926 100644 --- a/apps/webapp/app/components/LoginPageLayout.tsx +++ b/apps/webapp/app/components/LoginPageLayout.tsx @@ -72,7 +72,7 @@ export function LoginPageLayout({ {rightContent ?? ( <> <div className="flex h-full flex-col items-center justify-center px-16"> - <Header3 className="relative text-center text-2xl font-normal leading-8 text-text-dimmed transition before:relative before:right-1 before:top-0 before:text-6xl before:text-charcoal-750 before:content-['❝'] lg-height:text-xl md-height:text-lg"> + <Header3 className="relative text-center text-2xl font-normal leading-8 text-text-dimmed transition before:relative before:right-1 before:top-0 before:text-6xl before:text-grid-bright before:content-['❝'] dark:before:text-charcoal-750 lg-height:text-xl md-height:text-lg"> {randomQuote?.quote} </Header3> <Paragraph className="mt-4 text-text-dimmed/60">{randomQuote?.person}</Paragraph> diff --git a/apps/webapp/app/components/admin/debugTooltip.tsx b/apps/webapp/app/components/admin/debugTooltip.tsx index 1729bfa740c..8e7d6b77b71 100644 --- a/apps/webapp/app/components/admin/debugTooltip.tsx +++ b/apps/webapp/app/components/admin/debugTooltip.tsx @@ -45,7 +45,7 @@ function Content({ children }: { children: React.ReactNode }) { const user = useUser(); return ( - <div className="flex flex-col gap-2 divide-y divide-slate-700"> + <div className="flex flex-col gap-2 divide-y divide-grid-bright"> <Property.Table> <Property.Item> <Property.Label>User ID</Property.Label> diff --git a/apps/webapp/app/components/billing/AnimatedOrgBannerBar.tsx b/apps/webapp/app/components/billing/AnimatedOrgBannerBar.tsx index b0f11b7eba3..17eef91ad5c 100644 --- a/apps/webapp/app/components/billing/AnimatedOrgBannerBar.tsx +++ b/apps/webapp/app/components/billing/AnimatedOrgBannerBar.tsx @@ -40,11 +40,16 @@ export function AnimatedOrgBannerBar({ <div className="flex items-center gap-2"> <Icon icon={ExclamationCircleIcon} - className={cn("h-5 w-5", variant === "warning" ? "text-amber-400" : "text-error")} + className={cn( + "h-5 w-5", + variant === "warning" ? "text-amber-400 light:text-amber-600" : "text-error" + )} /> <Paragraph variant="small" - className={variant === "warning" ? "text-amber-200" : "text-error"} + className={ + variant === "warning" ? "text-amber-700 dark:text-amber-200" : "text-error" + } > {children} </Paragraph> diff --git a/apps/webapp/app/components/billing/BillingLimitRecoveryPanel.tsx b/apps/webapp/app/components/billing/BillingLimitRecoveryPanel.tsx index 53b3c1695de..1dfcdd73707 100644 --- a/apps/webapp/app/components/billing/BillingLimitRecoveryPanel.tsx +++ b/apps/webapp/app/components/billing/BillingLimitRecoveryPanel.tsx @@ -113,7 +113,7 @@ export function BillingLimitRecoveryPanel({ <div className="border-b border-grid-dimmed pb-3"> <Header2 spacing>Action required</Header2> <Callout variant="warning"> - <Paragraph variant="small" className="text-yellow-200"> + <Paragraph variant="small" className="text-callout-warning-text"> {isGrace ? ( <> Your organization has reached its billing limit. Processing is paused and new runs diff --git a/apps/webapp/app/components/billing/OrgBanner.tsx b/apps/webapp/app/components/billing/OrgBanner.tsx index df86f1471f7..03214bab7ec 100644 --- a/apps/webapp/app/components/billing/OrgBanner.tsx +++ b/apps/webapp/app/components/billing/OrgBanner.tsx @@ -151,8 +151,14 @@ function NoLimitConfiguredBanner() { variant="warning" action={ canManageBillingLimits ? ( - <LinkButton variant="tertiary/small" to={v3BillingLimitsPath(organization)}> - Configure billing limit + <LinkButton + variant="tertiary/small" + className="system:border-transparent system:bg-warning system:transition system:group-hover/button:bg-warning system:group-hover/button:brightness-90" + to={v3BillingLimitsPath(organization)} + > + <span className="mx-auto grow self-center truncate text-text-bright system:text-white"> + Configure billing limit + </span> </LinkButton> ) : undefined } diff --git a/apps/webapp/app/components/billing/UsageBar.tsx b/apps/webapp/app/components/billing/UsageBar.tsx index 49346492287..fcb6377757c 100644 --- a/apps/webapp/app/components/billing/UsageBar.tsx +++ b/apps/webapp/app/components/billing/UsageBar.tsx @@ -67,7 +67,7 @@ export function UsageBar({ current, billingLimit, tierLimit, isPaying }: UsageBa animate={{ width: tierRunLimitPercentage + "%" }} transition={{ duration: 1.5, type: "spring" }} style={{ width: `${tierRunLimitPercentage}%` }} - className="absolute h-3 rounded-l-sm bg-green-900/50" + className="absolute h-3 rounded-l-sm bg-green-900/20" > <Legend text={isPaying ? `Included usage:` : `Tier limit:`} diff --git a/apps/webapp/app/components/code/ChartConfigPanel.tsx b/apps/webapp/app/components/code/ChartConfigPanel.tsx index 1c915c2a449..7711f063d55 100644 --- a/apps/webapp/app/components/code/ChartConfigPanel.tsx +++ b/apps/webapp/app/components/code/ChartConfigPanel.tsx @@ -572,7 +572,7 @@ function SeriesColorPicker({ title="Change series color" > <span - className="block h-4 w-4 rounded-full border border-white/30" + className="block h-4 w-4 rounded-full border border-text-bright/30" style={{ backgroundColor: color }} /> </button> @@ -587,7 +587,7 @@ function SeriesColorPicker({ onColorChange(c); setOpen(false); }} - className="group/swatch flex h-6 w-6 items-center justify-center rounded-full border border-white/30" + className="group/swatch flex h-6 w-6 items-center justify-center rounded-full border border-text-bright/30" style={{ backgroundColor: c }} title={c} > diff --git a/apps/webapp/app/components/code/TSQLEditor.tsx b/apps/webapp/app/components/code/TSQLEditor.tsx index 3b4bf202954..0ddc7cf12d5 100644 --- a/apps/webapp/app/components/code/TSQLEditor.tsx +++ b/apps/webapp/app/components/code/TSQLEditor.tsx @@ -284,7 +284,7 @@ export function TSQLEditor(opts: TSQLEditorProps) { }} /> {showButtons && ( - <div className="absolute right-0 top-0 z-10 flex items-center justify-end bg-background-deep/80 p-1.5"> + <div className="absolute right-0 top-0 z-10 flex items-center justify-end bg-background-deep/80 light:bg-transparent p-1.5"> {additionalActions && additionalActions} {showFormatButton && ( <Button diff --git a/apps/webapp/app/components/code/codeMirrorTheme.ts b/apps/webapp/app/components/code/codeMirrorTheme.ts index f32ea560b2d..783d28f1fc0 100644 --- a/apps/webapp/app/components/code/codeMirrorTheme.ts +++ b/apps/webapp/app/components/code/codeMirrorTheme.ts @@ -84,7 +84,10 @@ export function darkTheme(): Extension { backgroundColor: "var(--color-editor-search-match-selected)", }, - ".cm-activeLine": { backgroundColor: highlightBackground }, + // Only highlight the active line while the editor has focus - on an + // unfocused editor the resting cursor paints a phantom gray bar + ".cm-activeLine": { backgroundColor: "transparent" }, + "&.cm-focused .cm-activeLine": { backgroundColor: highlightBackground }, ".cm-selectionMatch": { backgroundColor: "var(--color-editor-selection-match)" }, "&.cm-focused .cm-matchingBracket, &.cm-focused .cm-nonmatchingBracket": { @@ -99,6 +102,9 @@ export function darkTheme(): Extension { }, ".cm-activeLineGutter": { + backgroundColor: "transparent", + }, + "&.cm-focused .cm-activeLineGutter": { backgroundColor: highlightBackground, }, diff --git a/apps/webapp/app/components/environments/EnvironmentLabel.tsx b/apps/webapp/app/components/environments/EnvironmentLabel.tsx index 6f253676e04..1d34121403c 100644 --- a/apps/webapp/app/components/environments/EnvironmentLabel.tsx +++ b/apps/webapp/app/components/environments/EnvironmentLabel.tsx @@ -119,7 +119,8 @@ export function EnvironmentLabel({ ref={spanRef} className={cn( truncate ? "truncate" : "overflow-hidden whitespace-nowrap", - "text-left", + // system-mono-label: System themes uncolor the name, only the icon stays tinted + "text-left system-mono-label", environmentTextClassName(environment), className )} @@ -135,7 +136,10 @@ export function EnvironmentLabel({ asChild button={content} content={ - <span ref={spanRef} className={cn("text-left", environmentTextClassName(environment))}> + <span + ref={spanRef} + className={cn("text-left system-mono-label", environmentTextClassName(environment))} + > {text} </span> } @@ -151,7 +155,11 @@ export function EnvironmentLabel({ } export function EnvironmentSlug({ environment }: { environment: Environment & { slug: string } }) { - return <span className={environmentTextClassName(environment)}>{environment.slug}</span>; + return ( + <span className={cn("system-mono-label", environmentTextClassName(environment))}> + {environment.slug} + </span> + ); } export function environmentTitle(environment: Environment, username?: string) { diff --git a/apps/webapp/app/components/errors/ConfigureErrorAlerts.tsx b/apps/webapp/app/components/errors/ConfigureErrorAlerts.tsx index d6affafd3b7..b126e77b34b 100644 --- a/apps/webapp/app/components/errors/ConfigureErrorAlerts.tsx +++ b/apps/webapp/app/components/errors/ConfigureErrorAlerts.tsx @@ -1,12 +1,7 @@ import { getFormProps, getInputProps, useForm } from "@conform-to/react"; +import { GlobeLinesIcon } from "~/assets/icons/GlobeLinesIcon"; import { parseWithZod } from "@conform-to/zod"; -import { - EnvelopeIcon, - GlobeAltIcon, - HashtagIcon, - LockClosedIcon, - XMarkIcon, -} from "@heroicons/react/20/solid"; +import { EnvelopeIcon, HashtagIcon, LockClosedIcon, XMarkIcon } from "@heroicons/react/20/solid"; import { BellAlertIcon } from "@heroicons/react/24/solid"; import { useFetcher, useNavigate } from "@remix-run/react"; import { SlackIcon } from "@trigger.dev/companyicons"; @@ -319,7 +314,7 @@ export function ConfigureErrorAlerts({ placeholder={ index === 0 ? "https://example.com/webhook" : "Add another webhook URL" } - icon={GlobeAltIcon} + icon={GlobeLinesIcon} onChange={(e) => { webhookFieldValues.current[index] = e.target.value; if ( diff --git a/apps/webapp/app/components/errors/ErrorStatusBadge.tsx b/apps/webapp/app/components/errors/ErrorStatusBadge.tsx index 571a209ddf1..aad2bb2f1f2 100644 --- a/apps/webapp/app/components/errors/ErrorStatusBadge.tsx +++ b/apps/webapp/app/components/errors/ErrorStatusBadge.tsx @@ -1,10 +1,19 @@ import { type ErrorGroupStatus } from "@trigger.dev/database"; import { cn } from "~/utils/cn"; -const styles: Record<ErrorGroupStatus, string> = { +// Two prominence levels: `subtle` keeps the tinted look for dense lists, +// `bright` fills the chip with the status color in the System themes for +// solitary placements (detail headers). Classic always uses the tinted look. +const subtleStyles: Record<ErrorGroupStatus, string> = { UNRESOLVED: "bg-error/10 text-error", RESOLVED: "bg-success/10 text-success", - IGNORED: "bg-blue-500/10 text-blue-400", + IGNORED: "bg-blue-500/10 text-blue-400 system:text-blue-500", +}; + +const brightStyles: Record<ErrorGroupStatus, string> = { + UNRESOLVED: "bg-error/10 text-error system:bg-error system:text-white", + RESOLVED: "bg-success/10 text-success system:bg-success system:text-white", + IGNORED: "bg-blue-500/10 text-blue-400 system:bg-blue-500 system:text-white", }; const labels: Record<ErrorGroupStatus, string> = { @@ -16,15 +25,17 @@ const labels: Record<ErrorGroupStatus, string> = { export function ErrorStatusBadge({ status, className, + prominence = "subtle", }: { status: ErrorGroupStatus; className?: string; + prominence?: "subtle" | "bright"; }) { return ( <span className={cn( - "inline-flex items-center rounded px-2 py-0.5 text-xs font-medium", - styles[status], + "contrast-chip inline-flex items-center rounded px-2 py-0.5 text-xs font-medium", + (prominence === "bright" ? brightStyles : subtleStyles)[status], className )} > diff --git a/apps/webapp/app/components/integrations/VercelOnboardingModal.tsx b/apps/webapp/app/components/integrations/VercelOnboardingModal.tsx index cd5f9eee505..bda62a1b32f 100644 --- a/apps/webapp/app/components/integrations/VercelOnboardingModal.tsx +++ b/apps/webapp/app/components/integrations/VercelOnboardingModal.tsx @@ -17,7 +17,7 @@ import { InputGroup } from "~/components/primitives/InputGroup"; import { Label } from "~/components/primitives/Label"; import { Paragraph } from "~/components/primitives/Paragraph"; import { Select, SelectItem } from "~/components/primitives/Select"; -import { SpinnerWhite } from "~/components/primitives/Spinner"; +import { Spinner, SpinnerWhite } from "~/components/primitives/Spinner"; import { Switch } from "~/components/primitives/Switch"; import { Tooltip, @@ -755,7 +755,7 @@ export function VercelOnboardingModal({ </div> </DialogHeader> <div className="flex items-center justify-center py-8"> - <SpinnerWhite className="size-6" /> + <Spinner color="blue" className="size-6" /> </div> </DialogContent> </Dialog> diff --git a/apps/webapp/app/components/navigation/SideMenuHeader.tsx b/apps/webapp/app/components/navigation/SideMenuHeader.tsx index 8ccbfa7de70..698ab1a2326 100644 --- a/apps/webapp/app/components/navigation/SideMenuHeader.tsx +++ b/apps/webapp/app/components/navigation/SideMenuHeader.tsx @@ -1,8 +1,7 @@ import { useNavigation } from "@remix-run/react"; import { useEffect, useState } from "react"; import { motion } from "framer-motion"; -import { Popover, PopoverContent, PopoverCustomTrigger } from "../primitives/Popover"; -import { EllipsisHorizontalIcon } from "@heroicons/react/20/solid"; +import { Popover, PopoverContent, PopoverEllipseTrigger } from "../primitives/Popover"; export function SideMenuHeader({ title, @@ -30,7 +29,7 @@ export function SideMenuHeader({ return ( <motion.div - className="group flex h-4 items-center justify-between overflow-hidden pl-1.5" + className="group/side-header flex h-4 items-center justify-between overflow-hidden pl-1.5" initial={false} animate={{ opacity: hasCollapsedTitle ? 1 : isCollapsed ? 0 : 1, @@ -46,9 +45,12 @@ export function SideMenuHeader({ </h2> {children !== undefined ? ( <Popover onOpenChange={(open) => setHeaderMenuOpen(open)} open={isHeaderMenuOpen}> - <PopoverCustomTrigger className="p-1"> - <EllipsisHorizontalIcon className="h-4 w-4 text-text-faint transition group-hover:text-text-bright" /> - </PopoverCustomTrigger> + <PopoverEllipseTrigger + isOpen={isHeaderMenuOpen} + variant="ghost" + orientation="horizontal" + className="group-hover/side-header:text-text-bright" + /> <PopoverContent className="min-w-max overflow-y-auto p-0 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-surface-control" align="start" diff --git a/apps/webapp/app/components/navigation/SideMenuItem.tsx b/apps/webapp/app/components/navigation/SideMenuItem.tsx index d162c064620..9a06c933d97 100644 --- a/apps/webapp/app/components/navigation/SideMenuItem.tsx +++ b/apps/webapp/app/components/navigation/SideMenuItem.tsx @@ -135,7 +135,10 @@ export function SideMenuItem({ icon={icon} className={cn( "size-5 shrink-0", - isActive ? activeIconColor : (inactiveIconColor ?? "text-text-dimmed"), + // side-menu-active-icon: System themes neutralize the accent (see tailwind.css) + isActive + ? cn(activeIconColor, "side-menu-active-icon") + : (inactiveIconColor ?? "text-text-dimmed"), !isActive && !disableIconHover && "group-hover/menuitem:text-text-bright group-hover/menulink:text-text-bright", diff --git a/apps/webapp/app/components/navigation/TreeConnectors.tsx b/apps/webapp/app/components/navigation/TreeConnectors.tsx index 8479179a491..25e8212b899 100644 --- a/apps/webapp/app/components/navigation/TreeConnectors.tsx +++ b/apps/webapp/app/components/navigation/TreeConnectors.tsx @@ -5,7 +5,7 @@ import { cn } from "~/utils/cn"; export function TreeConnectorBranch({ className }: { className?: string }) { return ( <svg - className={cn("overflow-visible", className, "text-charcoal-600")} + className={cn("overflow-visible", className, "text-border-bright")} viewBox="0 0 20 20" fill="none" > @@ -18,7 +18,7 @@ export function TreeConnectorBranch({ className }: { className?: string }) { export function TreeConnectorEnd({ className }: { className?: string }) { return ( <svg - className={cn("overflow-visible", className, "text-charcoal-600")} + className={cn("overflow-visible", className, "text-border-bright")} viewBox="0 0 20 20" fill="none" > diff --git a/apps/webapp/app/components/primitives/AppliedFilter.tsx b/apps/webapp/app/components/primitives/AppliedFilter.tsx index c8fbc4d3ac3..6390a5826d6 100644 --- a/apps/webapp/app/components/primitives/AppliedFilter.tsx +++ b/apps/webapp/app/components/primitives/AppliedFilter.tsx @@ -4,11 +4,11 @@ import { cn } from "~/utils/cn"; const variants = { "secondary/small": { - box: "h-6 bg-secondary rounded pl-1.5 gap-1.5 text-xs divide-x divide-black/15 group-hover:bg-surface-control group-hover:border-border-brighter text-text-bright border border-border-bright", + box: "h-6 bg-secondary rounded pl-1.5 gap-1.5 text-xs divide-x divide-black/15 shadow-xs group-hover:bg-background-raised text-text-bright border border-border-bright/50", clear: "size-6 text-text-bright hover:text-text-bright transition-colors", }, "tertiary/small": { - box: "h-6 bg-tertiary rounded pl-1.5 gap-1.5 text-xs divide-x divide-black/15 group-hover:bg-surface-control", + box: "h-6 bg-tertiary rounded pl-1.5 gap-1.5 text-xs divide-x divide-black/15 group-hover:bg-background-raised", clear: "size-6 text-text-dimmed hover:text-text-bright transition-colors", }, "minimal/medium": { diff --git a/apps/webapp/app/components/primitives/Avatar.tsx b/apps/webapp/app/components/primitives/Avatar.tsx index 9fc6832fcc4..728626271f4 100644 --- a/apps/webapp/app/components/primitives/Avatar.tsx +++ b/apps/webapp/app/components/primitives/Avatar.tsx @@ -1,10 +1,10 @@ +import { GlobeLinesIcon } from "~/assets/icons/GlobeLinesIcon"; import { BoltIcon, BuildingOffice2Icon, CodeBracketSquareIcon, FaceSmileIcon, FireIcon, - GlobeAltIcon, RocketLaunchIcon, StarIcon, } from "@heroicons/react/20/solid"; @@ -117,6 +117,16 @@ function styleFromSize(size: number) { }; } +// Bright tiles (Yellow, Orange) need dark letters for contrast; the rest read +// best with white. +function letterColorForBackground(hex: string): string { + const match = /^#?([0-9a-f]{6})$/i.exec(hex); + if (!match) return "#fff"; + const n = parseInt(match[1], 16); + const luminance = 0.299 * ((n >> 16) & 255) + 0.587 * ((n >> 8) & 255) + 0.114 * (n & 255); + return luminance > 140 ? "#272A2E" : "#fff"; +} + function AvatarLetters({ avatar, size, @@ -132,15 +142,13 @@ function AvatarLetters({ const style = { backgroundColor: avatar.hex, + color: letterColorForBackground(avatar.hex), }; const scaleFactor = includePadding ? 0.8 : 1; return ( - <span - className="grid shrink-0 place-items-center overflow-hidden text-charcoal-750" - style={styleFromSize(size)} - > + <span className="grid shrink-0 place-items-center overflow-hidden" style={styleFromSize(size)}> {/* This is the square container */} <span className={cn( @@ -185,7 +193,7 @@ function AvatarImage({ avatar, size }: { avatar: ImageAvatar; size: number }) { if (!avatar.url) { return ( <span className="grid shrink-0 place-items-center" style={styleFromSize(size)}> - <GlobeAltIcon className="size-[90%] text-text-dimmed" /> + <GlobeLinesIcon className="size-[90%] text-text-dimmed" /> </span> ); } diff --git a/apps/webapp/app/components/primitives/Badge.tsx b/apps/webapp/app/components/primitives/Badge.tsx index f21a544a4cd..15733bcc647 100644 --- a/apps/webapp/app/components/primitives/Badge.tsx +++ b/apps/webapp/app/components/primitives/Badge.tsx @@ -5,13 +5,13 @@ const variants = { default: "grid place-items-center rounded-full px-2 h-5 tracking-wider text-xxs bg-background-hover text-text-bright uppercase whitespace-nowrap", "extra-small": - "grid place-items-center border border-border-bright rounded-sm px-1 h-4 text-xxs bg-background-bright text-blue-500 whitespace-nowrap", + "grid place-items-center border border-border-bright rounded-sm px-1 h-4 text-xxs bg-background-bright text-blue-500 system:border-transparent system:bg-blue-500/10 system:text-blue-500 whitespace-nowrap", small: - "grid place-items-center border border-border-bright rounded-sm px-1 h-5 text-xs bg-background-bright text-blue-500 whitespace-nowrap", + "grid place-items-center border border-border-bright rounded-sm px-1 h-5 text-xs bg-background-bright text-blue-500 system:border-transparent system:bg-blue-500/10 system:text-blue-500 whitespace-nowrap", "outline-rounded": "grid place-items-center rounded-full px-1 h-4 tracking-wider text-xxs border border-blue-500 text-blue-500 uppercase whitespace-nowrap", rounded: - "grid place-items-center rounded-full px-1.5 h-4 text-xxs border bg-blue-600 text-text-bright uppercase whitespace-nowrap", + "grid place-items-center rounded-full px-1.5 h-4 text-xxs border bg-blue-600 text-text-bright system:border-transparent system:text-white uppercase whitespace-nowrap", }; type BadgeProps = React.HTMLAttributes<HTMLDivElement> & { diff --git a/apps/webapp/app/components/primitives/BreadcrumbIcon.tsx b/apps/webapp/app/components/primitives/BreadcrumbIcon.tsx index b5dfb55ba7e..7d7fae46be0 100644 --- a/apps/webapp/app/components/primitives/BreadcrumbIcon.tsx +++ b/apps/webapp/app/components/primitives/BreadcrumbIcon.tsx @@ -3,7 +3,7 @@ import { cn } from "~/utils/cn"; export function BreadcrumbIcon({ className }: { className?: string }) { return ( <svg - className={cn("text-charcoal-600", className)} + className={cn("text-border-bright", className)} width="9" height="20" viewBox="0 0 9 26" diff --git a/apps/webapp/app/components/primitives/Buttons.tsx b/apps/webapp/app/components/primitives/Buttons.tsx index 792f73eb595..78b201b90e0 100644 --- a/apps/webapp/app/components/primitives/Buttons.tsx +++ b/apps/webapp/app/components/primitives/Buttons.tsx @@ -59,18 +59,17 @@ type Size = keyof typeof sizes; const theme = { primary: { - textColor: - "text-text-bright group-hover/button:text-white transition group-disabled/button:text-text-dimmed", + textColor: "text-white transition group-disabled/button:text-white/60", button: "bg-indigo-600 border border-indigo-500 group-hover/button:bg-indigo-500 group-hover/button:border-indigo-400 group-disabled/button:opacity-50 group-disabled/button:bg-indigo-600 group-disabled/button:border-indigo-500 group-disabled/button:pointer-events-none", shortcut: - "border-text-bright/40 text-text-bright group-hover/button:border-text-bright/60 group-hover/button:text-text-bright", - icon: "text-text-bright", + "border-white/40 text-white group-hover/button:border-white/60 group-hover/button:text-white", + icon: "text-white", }, secondary: { textColor: "text-text-bright transition group-disabled/button:text-text-dimmed/80", button: - "bg-secondary group-hover/button:bg-surface-control group-hover/button:border-border-brighter border border-border-bright group-disabled/button:bg-secondary group-disabled/button:opacity-60 group-disabled/button:pointer-events-none", + "bg-secondary border border-border-bright/50 shadow-xs group-hover/button:bg-background-raised group-disabled/button:bg-secondary group-disabled/button:opacity-60 group-disabled/button:pointer-events-none", shortcut: "border-text-dimmed/40 text-text-dimmed group-hover/button:text-text-bright group-hover/button:border-text-dimmed", icon: "text-text-bright", @@ -92,12 +91,11 @@ const theme = { icon: "text-text-dimmed", }, danger: { - textColor: - "text-text-bright group-hover/button:text-white transition group-disabled/button:text-text-bright/80", + textColor: "text-white transition group-disabled/button:text-white/80", button: "bg-error group-hover/button:bg-rose-500 disabled:opacity-50 group-disabled/button:bg-error group-disabled/button:pointer-events-none", - shortcut: "border-text-bright text-text-bright group-hover/button:border-text-bright/60", - icon: "text-text-bright", + shortcut: "border-white text-white group-hover/button:border-white/60", + icon: "text-white", }, warning: { textColor: "text-warning transition group-disabled/button:text-warning/60", @@ -107,9 +105,11 @@ const theme = { icon: "text-warning", }, docs: { - textColor: "text-blue-200/70 transition group-disabled/button:text-text-dimmed/80", + textColor: + // System themes: monochrome label, the book icon keeps the blue + "text-callout-docs-text/70 system:text-text-bright transition group-disabled/button:text-text-dimmed/80", button: - "bg-background-raised border border-border-bright/50 shadow-sm group-hover/button:bg-secondary group-disabled/button:bg-tertiary group-disabled/button:opacity-60 group-disabled/button:pointer-events-none", + "bg-secondary border border-border-bright/50 shadow-xs group-hover/button:bg-background-raised group-disabled/button:bg-tertiary group-disabled/button:opacity-60 group-disabled/button:pointer-events-none", shortcut: "border-text-dimmed/40 text-text-dimmed group-hover/button:text-text-bright group-hover/button:border-text-dimmed", icon: "text-blue-500", @@ -223,7 +223,7 @@ const variant = { }; const allVariants = { - $all: "font-normal text-center font-sans justify-center items-center shrink-0 transition duration-150 rounded-[3px] select-none group-focus/button:outline-hidden group-disabled/button:opacity-75 group-disabled/button:pointer-events-none focus-custom", + $all: "cursor-pointer font-normal text-center font-sans justify-center items-center shrink-0 transition duration-150 rounded-[3px] select-none group-focus/button:outline-hidden group-disabled/button:opacity-75 group-disabled/button:pointer-events-none focus-custom", variant: variant, }; diff --git a/apps/webapp/app/components/primitives/Checkbox.tsx b/apps/webapp/app/components/primitives/Checkbox.tsx index 75850b304d9..da63e2ed993 100644 --- a/apps/webapp/app/components/primitives/Checkbox.tsx +++ b/apps/webapp/app/components/primitives/Checkbox.tsx @@ -139,7 +139,11 @@ export const CheckboxWithLabel = React.forwardRef<HTMLInputElement, CheckboxProp className={cn( inputPositionClasses, props.readOnly || disabled ? "cursor-default" : "cursor-pointer", - "read-only:border-border-bright disabled:border-border-bright rounded-sm border border-border-bright bg-transparent transition checked:bg-indigo-500! read-only:bg-background-raised! group-hover:bg-background-deep checked:group-hover:bg-indigo-500 group-focus:ring-1 focus:ring-indigo-500 focus:ring-offset-0 focus:ring-offset-transparent focus-visible:outline-hidden focus-visible:ring-indigo-500 disabled:bg-background-raised!" + // NB: don't use the `read-only:` variant here — checkboxes always + // match :read-only, so it would override the checked style. + "rounded-sm border border-border-bright bg-transparent transition checked:bg-indigo-500! group-hover:bg-background-deep checked:group-hover:bg-indigo-500 group-focus:ring-1 focus:ring-indigo-500 focus:ring-offset-0 focus:ring-offset-transparent focus-visible:outline-hidden focus-visible:ring-indigo-500", + (isDisabled || props.readOnly) && + "bg-background-raised! checked:bg-background-raised! checked:group-hover:bg-background-raised! group-hover:bg-background-raised!" )} id={id} ref={ref} @@ -189,7 +193,13 @@ export const Checkbox = forwardRef<HTMLInputElement, SimpleCheckboxProps>( : props.readOnly ? "cursor-default" : "cursor-pointer", - "read-only:border-border-bright disabled:border-border-bright disabled:opacity-50 rounded-sm border border-border-bright bg-transparent transition checked:bg-indigo-500! read-only:bg-background-raised! group-hover:bg-background-deep checked:group-hover:bg-indigo-500 group-focus:ring-1 focus:ring-indigo-500 focus:ring-offset-0 focus:ring-offset-transparent focus-visible:outline-hidden focus-visible:ring-indigo-500 disabled:bg-background-raised!" + // NB: don't use the `read-only:` variant here — checkboxes always + // match :read-only, so it would override the checked style. + "rounded-sm border border-border-bright bg-transparent transition checked:bg-indigo-500! group-hover:bg-background-deep checked:group-hover:bg-indigo-500 group-focus:ring-1 focus:ring-indigo-500 focus:ring-offset-0 focus:ring-offset-transparent focus-visible:outline-hidden focus-visible:ring-indigo-500", + props.disabled && "opacity-50", + (props.disabled || props.readOnly) && + "bg-background-raised! checked:bg-background-raised! checked:group-hover:bg-background-raised! group-hover:bg-background-raised!", + className )} {...props} ref={ref} diff --git a/apps/webapp/app/components/primitives/ClientTabs.tsx b/apps/webapp/app/components/primitives/ClientTabs.tsx index 533a09c84e6..48757676d61 100644 --- a/apps/webapp/app/components/primitives/ClientTabs.tsx +++ b/apps/webapp/app/components/primitives/ClientTabs.tsx @@ -122,10 +122,10 @@ const ClientTabsTrigger = React.forwardRef< <motion.div layoutId={layoutId} transition={{ duration: 0.4, type: "spring" }} - className="absolute inset-0 rounded-[2px] border border-border-brightest/50 bg-surface-control" + className="absolute inset-0 rounded-[2px] border border-grid-bright bg-white dark:border-border-brightest/50 dark:bg-surface-control" /> ) : ( - <div className="absolute inset-0 rounded-[2px] border border-border-brightest/50 bg-surface-control" /> + <div className="absolute inset-0 rounded-[2px] border border-grid-bright bg-white dark:border-border-brightest/50 dark:bg-surface-control" /> ) ) : null} </TabsPrimitive.Trigger> @@ -175,7 +175,7 @@ const ClientTabsTrigger = React.forwardRef< ref={ref} tabIndex={0} className={cn( - "inline-flex items-center justify-center whitespace-nowrap border-r border-grid-bright px-2 text-sm transition-all first:pl-0 last:border-none focus-custom data-[state=active]:text-indigo-500 data-[state=inactive]:text-text-dimmed data-[state=inactive]:hover:text-text-bright disabled:pointer-events-none disabled:opacity-50", + "inline-flex items-center justify-center whitespace-nowrap border-r border-grid-bright px-2 text-sm transition-all first:pl-0 last:border-none focus-custom data-[state=active]:text-indigo-500 system:data-[state=active]:text-text-bright data-[state=inactive]:text-text-dimmed data-[state=inactive]:hover:text-text-bright disabled:pointer-events-none disabled:opacity-50", className )} {...props} diff --git a/apps/webapp/app/components/primitives/ClipboardField.tsx b/apps/webapp/app/components/primitives/ClipboardField.tsx index 7774f301218..ca5df717d34 100644 --- a/apps/webapp/app/components/primitives/ClipboardField.tsx +++ b/apps/webapp/app/components/primitives/ClipboardField.tsx @@ -5,7 +5,7 @@ import { CopyButton } from "./CopyButton"; const variants = { "primary/small": { container: - "flex items-center text-text-dimmed font-mono rounded border bg-background-hover text-xs transition hover:bg-background-raised focus-visible:outline-hidden focus-visible:ring-0 focus-visible:ring-offset-0 focus:border-transparent focus:outline-hidden focus:ring-0 focus:ring-transparent", + "flex items-center text-text-dimmed font-mono rounded border border-border-bright/50 shadow-xs bg-input-bg text-xs transition hover:bg-background-raised focus-visible:outline-hidden focus-visible:ring-0 focus-visible:ring-offset-0 focus:border-transparent focus:outline-hidden focus:ring-0 focus:ring-transparent", input: "bg-transparent border-0 text-xs px-2 w-auto rounded-l h-6 leading-6 focus:ring-transparent", buttonVariant: "primary" as const, @@ -14,7 +14,7 @@ const variants = { }, "secondary/small": { container: - "flex items-center text-text-dimmed font-mono rounded border bg-background-hover text-xs transition hover:bg-background-raised focus-visible:outline-hidden focus-visible:ring-0 focus-visible:ring-offset-0 focus:border-transparent focus:outline-hidden focus:ring-0 focus:ring-transparent", + "flex items-center text-text-dimmed font-mono rounded border border-border-bright/50 shadow-xs bg-input-bg text-xs transition hover:bg-background-raised focus-visible:outline-hidden focus-visible:ring-0 focus-visible:ring-offset-0 focus:border-transparent focus:outline-hidden focus:ring-0 focus:ring-transparent", input: "bg-transparent border-0 text-xs px-2 w-auto rounded-l h-6 leading-6 focus:ring-transparent", buttonVariant: "tertiary" as const, @@ -33,7 +33,7 @@ const variants = { }, "primary/medium": { container: - "flex items-center text-text-dimmed font-mono rounded border bg-background-hover text-sm transition hover:bg-background-raised focus-visible:outline-hidden focus-visible:ring-0 focus-visible:ring-offset-0 focus:border-transparent focus:outline-hidden focus:ring-0 focus:ring-transparent", + "flex items-center text-text-dimmed font-mono rounded border border-border-bright/50 shadow-xs bg-input-bg text-sm transition hover:bg-background-raised focus-visible:outline-hidden focus-visible:ring-0 focus-visible:ring-offset-0 focus:border-transparent focus:outline-hidden focus:ring-0 focus:ring-transparent", input: "bg-transparent border-0 text-sm px-3 w-auto rounded-l h-8 leading-6 focus:ring-transparent", buttonVariant: "primary" as const, @@ -42,7 +42,7 @@ const variants = { }, "secondary/medium": { container: - "flex items-center text-text-dimmed font-mono rounded bg-background-hover text-sm transition hover:bg-background-raised focus-visible:outline-hidden focus-visible:ring-0 focus-visible:ring-offset-0 focus:border-transparent focus:outline-hidden focus:ring-0 focus:ring-transparent", + "flex items-center text-text-dimmed font-mono rounded border border-border-bright/50 shadow-xs bg-input-bg text-sm transition hover:bg-background-raised focus-visible:outline-hidden focus-visible:ring-0 focus-visible:ring-offset-0 focus:border-transparent focus:outline-hidden focus:ring-0 focus:ring-transparent", input: "bg-transparent border-0 text-sm px-3 w-auto rounded-l h-8 leading-6 focus:ring-transparent", buttonVariant: "tertiary" as const, diff --git a/apps/webapp/app/components/primitives/DateField.tsx b/apps/webapp/app/components/primitives/DateField.tsx index 56cf5086493..a68616fc1b1 100644 --- a/apps/webapp/app/components/primitives/DateField.tsx +++ b/apps/webapp/app/components/primitives/DateField.tsx @@ -275,7 +275,7 @@ function DateSegmentGuide({ segment }: { segment: DateSegment }) { style={{ minWidth: minWidthForSegment(segment), }} - className={`group box-content rounded-sm px-0.5 text-right text-sm tabular-nums text-rose-500 outline-hidden ${ + className={`group box-content rounded-sm px-0.5 text-right text-sm tabular-nums outline-hidden ${ !segment.isEditable ? "text-text-faint" : "text-text-bright" }`} > diff --git a/apps/webapp/app/components/primitives/DateTime.tsx b/apps/webapp/app/components/primitives/DateTime.tsx index 355af397a11..3c1227e0c9a 100644 --- a/apps/webapp/app/components/primitives/DateTime.tsx +++ b/apps/webapp/app/components/primitives/DateTime.tsx @@ -1,4 +1,5 @@ -import { GlobeAltIcon, GlobeAmericasIcon } from "@heroicons/react/20/solid"; +import { GlobeAmericasIcon } from "@heroicons/react/20/solid"; +import { GlobeLinesIcon } from "~/assets/icons/GlobeLinesIcon"; import { useRouteLoaderData } from "@remix-run/react"; import { formatDistanceToNow } from "date-fns"; import { Laptop } from "lucide-react"; @@ -523,7 +524,7 @@ function TooltipContent({ title="UTC" dateTime={formatDateTime(realDate, "UTC", locales, true, true, true)} isoDateTime={formatDateTimeISO(realDate, "UTC")} - icon={<GlobeAltIcon className="size-4 text-blue-500" />} + icon={<GlobeLinesIcon className="size-4 text-blue-500" />} /> <DateTimeTooltipContent title="Local" diff --git a/apps/webapp/app/components/primitives/Input.tsx b/apps/webapp/app/components/primitives/Input.tsx index 4f20c0c11d6..5c3235a66c9 100644 --- a/apps/webapp/app/components/primitives/Input.tsx +++ b/apps/webapp/app/components/primitives/Input.tsx @@ -12,21 +12,21 @@ const inputBase = const variants = { large: { container: - "px-1 w-full h-10 rounded-[3px] border border-background-bright bg-background-hover hover:border-border-bright hover:bg-secondary", + "px-1 w-full h-10 rounded-[3px] border border-border-bright/50 shadow-xs bg-input-bg hover:bg-background-raised", input: "px-2 text-sm", iconSize: "size-4 ml-1", accessory: "pr-1", }, medium: { container: - "px-1 h-8 w-full rounded border border-background-bright bg-background-hover hover:border-border-bright hover:bg-secondary", + "px-1 h-8 w-full rounded border border-border-bright/50 shadow-xs bg-input-bg hover:bg-background-raised", input: "px-1.5 rounded text-sm", iconSize: "size-4 ml-0.5", accessory: "pr-1", }, small: { container: - "px-1 h-6 w-full rounded border border-background-bright bg-background-hover hover:border-border-bright hover:bg-secondary", + "px-1 h-6 w-full rounded border border-border-bright/50 shadow-xs bg-input-bg hover:bg-background-raised", input: "px-1 rounded text-xs", iconSize: "size-3 ml-0.5", accessory: "pr-0.5", @@ -39,7 +39,7 @@ const variants = { }, "secondary-small": { container: - "px-1 h-6 w-full rounded border border-border-bright hover:border-border-brighter bg-grid-dimmed hover:bg-secondary", + "px-1 h-6 w-full rounded border border-border-bright/50 shadow-xs bg-input-bg hover:bg-background-raised", input: "px-1 rounded text-xs", iconSize: "size-3 ml-0.5", accessory: "pr-0.5", diff --git a/apps/webapp/app/components/primitives/InputNumberStepper.tsx b/apps/webapp/app/components/primitives/InputNumberStepper.tsx index 6869b4424ff..11fbb676f2b 100644 --- a/apps/webapp/app/components/primitives/InputNumberStepper.tsx +++ b/apps/webapp/app/components/primitives/InputNumberStepper.tsx @@ -189,13 +189,14 @@ export function InputNumberStepper({ aria-label={`Decrease by ${step}`} className={cn( "flex items-center justify-center rounded border border-error/30 bg-error/20 transition", + "system:border-transparent system:bg-error system:transition system:hover:bg-error system:hover:brightness-90", size.button, "hover:border-error/50 hover:bg-error/30", "disabled:cursor-not-allowed disabled:opacity-40", "focus-visible:outline-hidden focus-visible:ring-2 focus-visible:ring-text-link" )} > - <MinusIcon className={cn("text-error", size.icon)} /> + <MinusIcon className={cn("text-error system:text-white", size.icon)} /> </button> <button @@ -206,13 +207,14 @@ export function InputNumberStepper({ aria-label={`Increase by ${step}`} className={cn( "flex items-center justify-center rounded border border-success/30 bg-success/10 transition", + "system:border-transparent system:bg-success system:transition system:hover:bg-success system:hover:brightness-90", size.button, "hover:border-success/40 hover:bg-success/20", "disabled:cursor-not-allowed disabled:opacity-40 disabled:hover:bg-transparent", "focus-visible:outline-hidden focus-visible:ring-2 focus-visible:ring-text-link" )} > - <PlusIcon className={cn("text-success", size.icon)} /> + <PlusIcon className={cn("text-success system:text-white", size.icon)} /> </button> </div> </div> diff --git a/apps/webapp/app/components/primitives/Popover.tsx b/apps/webapp/app/components/primitives/Popover.tsx index 21ff5cd9045..fccba52a89b 100644 --- a/apps/webapp/app/components/primitives/Popover.tsx +++ b/apps/webapp/app/components/primitives/Popover.tsx @@ -1,7 +1,7 @@ "use client"; import { CheckIcon } from "@heroicons/react/20/solid"; -import { EllipsisVerticalIcon } from "@heroicons/react/24/solid"; +import { EllipsisHorizontalIcon, EllipsisVerticalIcon } from "@heroicons/react/24/solid"; import * as PopoverPrimitive from "@radix-ui/react-popover"; import { Link } from "@remix-run/react"; import * as React from "react"; @@ -279,20 +279,29 @@ const popoverVerticalEllipseVariants = { "size-6 rounded border border-border-bright bg-secondary text-text-bright hover:bg-surface-control hover:border-border-brighter", icon: "size-4", }, + // No box/background — the icon inherits the trigger's text color, so callers + // can drive brightening from a parent hover (e.g. a section header). + ghost: { + trigger: "p-1 text-text-faint hover:text-text-bright", + icon: "size-4", + }, } as const; type PopoverVerticalEllipseVariant = keyof typeof popoverVerticalEllipseVariants; -function PopoverVerticalEllipseTrigger({ +function PopoverEllipseTrigger({ isOpen, variant = "minimal", + orientation = "vertical", className, ...props }: { isOpen?: boolean; variant?: PopoverVerticalEllipseVariant; + orientation?: "vertical" | "horizontal"; } & React.ComponentPropsWithoutRef<typeof PopoverTrigger>) { const styles = popoverVerticalEllipseVariants[variant]; + const Icon = orientation === "horizontal" ? EllipsisHorizontalIcon : EllipsisVerticalIcon; return ( <PopoverTrigger {...props} @@ -302,11 +311,14 @@ function PopoverVerticalEllipseTrigger({ className )} > - <EllipsisVerticalIcon className={cn(styles.icon, "transition")} /> + <Icon className={cn(styles.icon, "transition")} /> </PopoverTrigger> ); } +// Back-compat alias: the trigger now supports both orientations. +const PopoverVerticalEllipseTrigger = PopoverEllipseTrigger; + export { Popover, PopoverArrowTrigger, @@ -314,6 +326,7 @@ export { PopoverCustomTrigger, PopoverMenuItem, PopoverSectionHeader, + PopoverEllipseTrigger, PopoverSideMenuTrigger, PopoverTrigger, PopoverVerticalEllipseTrigger, diff --git a/apps/webapp/app/components/primitives/RadioButton.tsx b/apps/webapp/app/components/primitives/RadioButton.tsx index c52389f788f..d936a81b923 100644 --- a/apps/webapp/app/components/primitives/RadioButton.tsx +++ b/apps/webapp/app/components/primitives/RadioButton.tsx @@ -22,7 +22,7 @@ const variants = { }, "button/small": { button: - "flex items-center w-fit h-8 pl-2 pr-3 rounded-md border hover:data-[state=checked]:border-border-bright border-border-bright hover:border-border-bright transition data-disabled:opacity-70 data-disabled:hover:bg-transparent hover:data-[state=checked]:bg-white/4 data-[state=checked]:bg-white/4", + "flex items-center w-fit h-8 pl-2 pr-3 rounded-md border border-border-bright/50 shadow-xs bg-secondary transition hover:bg-background-raised data-disabled:opacity-70 data-disabled:hover:bg-secondary hover:data-[state=checked]:bg-text-bright/4 data-[state=checked]:bg-text-bright/4", label: "text-sm text-text-bright select-none", description: "text-text-dimmed", inputPosition: "mt-0", @@ -30,7 +30,7 @@ const variants = { }, button: { button: - "w-fit py-2 pl-3 pr-4 rounded border border-border-bright hover:bg-background-dimmed hover:border-border-brightest transition data-[state=checked]:bg-background-dimmed data-disabled:opacity-70", + "w-fit py-2 pl-3 pr-4 rounded border border-border-bright/50 shadow-xs bg-secondary hover:bg-background-raised transition data-[state=checked]:bg-background-dimmed data-disabled:opacity-70", label: "text-text-bright select-none", description: "text-text-dimmed", inputPosition: "mt-1", @@ -38,7 +38,7 @@ const variants = { }, description: { button: - "w-full p-2.5 hover:data-[state=checked]:bg-white/4 data-[state=checked]:bg-white/4 transition data-disabled:opacity-70 hover:border-border-bright border-border-bright hover:data-[state=checked]:border-border-bright border rounded-md", + "w-full p-2.5 rounded-md border border-border-bright/50 shadow-xs bg-secondary transition hover:bg-background-raised data-disabled:opacity-70 hover:data-[state=checked]:bg-text-bright/4 data-[state=checked]:bg-text-bright/4", label: "text-text-bright font-semibold -mt-0.5 text-left text-sm", description: "text-text-dimmed mt-0 text-left", inputPosition: "mt-0", @@ -46,7 +46,7 @@ const variants = { }, icon: { button: - "w-full p-2.5 pb-4 hover:bg-background-dimmed transition data-disabled:opacity-70 data-[state=checked]:bg-background-dimmed border-border-bright border rounded-sm", + "w-full p-2.5 pb-4 rounded-sm border border-border-bright/50 shadow-xs bg-secondary hover:bg-background-raised transition data-disabled:opacity-70 data-[state=checked]:bg-background-dimmed", label: "text-text-bright font-semibold -mt-1 text-left", description: "text-text-dimmed mt-0 text-left", inputPosition: "mt-0", @@ -82,7 +82,10 @@ export function RadioButtonCircle({ )} > <Circle - className={cn("size-2 fill-text-bright text-text-bright", innerCircleClassName)} + className={cn( + "size-2 fill-white text-white dark:fill-text-bright dark:text-text-bright", + innerCircleClassName + )} /> </div> )} @@ -136,7 +139,7 @@ export const RadioGroupItem = React.forwardRef< )} > <RadioGroupPrimitive.Indicator className="flex h-full w-full items-center justify-center rounded-full bg-indigo-600"> - <Circle className="size-2 fill-text-bright text-text-bright" /> + <Circle className="size-2 fill-white text-white dark:fill-text-bright dark:text-text-bright" /> </RadioGroupPrimitive.Indicator> </div> <div className={cn(icon ? "flex h-full flex-col justify-end" : "")}> diff --git a/apps/webapp/app/components/primitives/Resizable.tsx b/apps/webapp/app/components/primitives/Resizable.tsx index 59550332c26..0bd4f8e86d9 100644 --- a/apps/webapp/app/components/primitives/Resizable.tsx +++ b/apps/webapp/app/components/primitives/Resizable.tsx @@ -2,6 +2,8 @@ import React, { useRef } from "react"; import { PanelGroup, Panel, PanelResizer } from "@window-splitter/react"; +import { useTypedMatchesData } from "~/hooks/useTypedMatchData"; +import type { loader as rootLoader } from "~/root"; import { cn } from "~/utils/cn"; const ResizablePanelGroup = ({ className, ...props }: React.ComponentProps<typeof PanelGroup>) => ( @@ -14,7 +16,25 @@ const ResizablePanelGroup = ({ className, ...props }: React.ComponentProps<typeo /> ); -const ResizablePanel = Panel; +// react-window-splitter drives the collapse animation through @react-spring/rafz, +// which has timing/interaction issues with Firefox that produce visual glitches +// (alternating frames, panels stuck at min, panelHasSpace invariant violations), +// so the animation is dropped on Firefox. The browser check must agree between +// SSR and hydration (a client-only `navigator` check made the panel tree differ +// and shifted useIds), so it comes from the root loader's user-agent sniff. +const ResizablePanel = React.forwardRef< + React.ElementRef<typeof Panel>, + React.ComponentProps<typeof Panel> +>(function ResizablePanel({ collapseAnimation, ...props }, ref) { + const rootData = useTypedMatchesData<typeof rootLoader>({ id: "root" }); + return ( + <Panel + ref={ref} + collapseAnimation={rootData?.isFirefox ? undefined : collapseAnimation} + {...props} + /> + ); +}); const ResizableHandle = ({ withHandle = true, @@ -69,14 +89,8 @@ const ResizableHandle = ({ </PanelResizer> ); -// react-window-splitter drives the collapse animation through @react-spring/rafz, -// which has timing/interaction issues with Firefox that produce visual glitches -// (alternating frames, panels stuck at min, panelHasSpace invariant violations). -// Disable the animation on Firefox; it works correctly in Chromium and Safari. -const RESIZABLE_PANEL_ANIMATION = - typeof navigator !== "undefined" && /firefox/i.test(navigator.userAgent) - ? undefined - : ({ easing: "ease-in-out", duration: 300 } as const); +// Firefox filtering happens inside ResizablePanel (see above). +const RESIZABLE_PANEL_ANIMATION = { easing: "ease-in-out", duration: 300 } as const; const COLLAPSIBLE_HANDLE_CLASSNAME = "transition-opacity duration-200"; diff --git a/apps/webapp/app/components/primitives/SegmentedControl.tsx b/apps/webapp/app/components/primitives/SegmentedControl.tsx index de9a84b5c0c..0fdccdbbc42 100644 --- a/apps/webapp/app/components/primitives/SegmentedControl.tsx +++ b/apps/webapp/app/components/primitives/SegmentedControl.tsx @@ -24,10 +24,11 @@ const theme = { selected: "absolute inset-0 rounded-[2px] outline-solid outline-3 outline-primary", }, secondary: { - base: "bg-background-raised/50", + base: "bg-transparent dark:bg-background-raised/50", active: "text-text-bright", inactive: "text-text-dimmed transition hover:text-text-bright", - selected: "absolute inset-0 rounded bg-background-raised border border-border-bright", + selected: + "absolute inset-0 rounded bg-white border border-grid-bright dark:bg-background-raised dark:border-border-bright", }, }; diff --git a/apps/webapp/app/components/primitives/Select.tsx b/apps/webapp/app/components/primitives/Select.tsx index 0f592767ae6..31921ef9854 100644 --- a/apps/webapp/app/components/primitives/Select.tsx +++ b/apps/webapp/app/components/primitives/Select.tsx @@ -30,7 +30,7 @@ const style = { }, secondary: { button: - "bg-secondary focus-custom border border-border-bright hover:text-text-bright hover:border-border-brighter text-text-bright hover:bg-surface-control", + "bg-secondary focus-custom border border-border-bright/50 shadow-xs hover:text-text-bright text-text-bright hover:bg-background-raised", }, }; @@ -65,6 +65,7 @@ type Section<TItem> = { function isSection<TItem>(data: TItem[] | Section<TItem>[]): data is Section<TItem>[] { const firstItem = data[0]; + if (!firstItem) return false; return ( (firstItem as Section<TItem>).type === "section" && (firstItem as Section<TItem>).items !== undefined && diff --git a/apps/webapp/app/components/primitives/Slider.tsx b/apps/webapp/app/components/primitives/Slider.tsx index b5af54e2ba4..e3ab6cdcd4e 100644 --- a/apps/webapp/app/components/primitives/Slider.tsx +++ b/apps/webapp/app/components/primitives/Slider.tsx @@ -5,6 +5,18 @@ import type { RenderIcon } from "./Icon"; import { Icon } from "./Icon"; const variants = { + /* Quiet variant for settings rows: no hover box, no thumb halo */ + settings: { + container: "h-6 gap-1 rounded-sm px-1", + icons: "h-4 w-4 text-text-bright", + root: "h-4 grow", + track: "h-1 bg-grid-bright", + range: "bg-transparent", + // Matches the Switch thumb; the secondary-button hairline+shadow keeps the + // white dot visible on the light track + thumb: + "h-3 w-3 border border-border-bright bg-white shadow-sm dark:border-transparent dark:bg-charcoal-200 dark:shadow-none", + }, tertiary: { container: "h-6 gap-1 rounded-sm hover:bg-background-raised px-1", icons: "h-4 w-4 text-text-bright", @@ -24,7 +36,14 @@ export type SliderProps = ComponentProps<typeof RadixSlider.Root> & { variant: VariantName; }; -export function Slider({ variant, className, LeadingIcon, TrailingIcon, ...props }: SliderProps) { +export function Slider({ + variant, + className, + LeadingIcon, + TrailingIcon, + "aria-label": ariaLabel, + ...props +}: SliderProps) { const variation = variants[variant]; return ( <div className={cn("group flex items-center", variation.container)}> @@ -40,7 +59,9 @@ export function Slider({ variant, className, LeadingIcon, TrailingIcon, ...props <RadixSlider.Track className={cn("relative grow rounded-full", variation.track)}> <RadixSlider.Range className={cn("absolute h-full rounded-full", variation.range)} /> </RadixSlider.Track> + {/* The thumb is the role="slider" element, so the label lives here */} <RadixSlider.Thumb + aria-label={ariaLabel} className={cn( "block cursor-pointer rounded-full transition focus:outline-hidden", variation.thumb diff --git a/apps/webapp/app/components/primitives/Switch.tsx b/apps/webapp/app/components/primitives/Switch.tsx index 96943e9bebb..e07187176a8 100644 --- a/apps/webapp/app/components/primitives/Switch.tsx +++ b/apps/webapp/app/components/primitives/Switch.tsx @@ -36,7 +36,7 @@ const variations = { "secondary/small": { container: cn( small.container, - "border border-border-bright hover:border-border-brighter bg-secondary hover:bg-surface-control" + "border border-border-bright/50 shadow-xs bg-secondary hover:bg-background-raised" ), root: cn( small.root, @@ -98,7 +98,7 @@ export const Switch = React.forwardRef<React.ElementRef<typeof SwitchPrimitives. <SwitchPrimitives.Thumb className={cn( thumb, - "pointer-events-none block rounded-full bg-charcoal-200 transition group-data-[state=checked]:bg-text-bright" + "pointer-events-none block rounded-full bg-white transition dark:bg-charcoal-200 dark:group-data-[state=checked]:bg-text-bright" )} /> </div> diff --git a/apps/webapp/app/components/primitives/UsageSparkline.tsx b/apps/webapp/app/components/primitives/UsageSparkline.tsx index 89c471a405e..ac80fff01ec 100644 --- a/apps/webapp/app/components/primitives/UsageSparkline.tsx +++ b/apps/webapp/app/components/primitives/UsageSparkline.tsx @@ -106,6 +106,8 @@ export function UsageSparkline({ </BarChart> </ResponsiveContainer> </div> + {/* No system-mono-label on the total: its color is often the only signal + (threshold breaches), there is no icon to carry it */} {hideTotal ? null : ( <span className={cn("-mt-1 text-xs tabular-nums", totalClassName)}> {formatTotal ? formatTotal(total) : total.toLocaleString()} diff --git a/apps/webapp/app/components/primitives/charts/Chart.tsx b/apps/webapp/app/components/primitives/charts/Chart.tsx index 932eaa516b1..d19452c0bc3 100644 --- a/apps/webapp/app/components/primitives/charts/Chart.tsx +++ b/apps/webapp/app/components/primitives/charts/Chart.tsx @@ -5,7 +5,9 @@ import { AnimatedNumber } from "../AnimatedNumber"; import TooltipPortal from "../TooltipPortal"; // Format: { THEME_NAME: CSS_SELECTOR } -const THEMES = { light: "", dark: '[data-theme="dark"]' } as const; +// dark covers Classic too; :is() keeps it one scoped selector when the prefix is +// interpolated as `${prefix} [data-chart=...]` below (a comma would break scoping). +const THEMES = { light: "", dark: ':is([data-theme="dark"], [data-theme="classic"])' } as const; export type ChartState = "loading" | "noData" | "invalid" | "loaded" | undefined; diff --git a/apps/webapp/app/components/primitives/charts/ChartZoom.tsx b/apps/webapp/app/components/primitives/charts/ChartZoom.tsx index 375a133f445..4be839f3f68 100644 --- a/apps/webapp/app/components/primitives/charts/ChartZoom.tsx +++ b/apps/webapp/app/components/primitives/charts/ChartZoom.tsx @@ -126,8 +126,8 @@ export function ZoomTooltip({ className={cn( "absolute whitespace-nowrap rounded border px-2 py-1 text-xxs tabular-nums", invalidSelection - ? "border-amber-800 bg-amber-950 text-amber-400" - : "border-blue-800 bg-[#1B2334] text-blue-400" + ? "border-amber-800 bg-amber-950 text-amber-400 light:border-amber-300 light:bg-amber-50 light:text-amber-700" + : "border-blue-800 bg-[#1B2334] text-blue-400 light:border-blue-300 light:bg-blue-50 light:text-blue-700" )} style={{ left: coordinate?.x, @@ -140,8 +140,8 @@ export function ZoomTooltip({ className={cn( "absolute top-[-5px] left-1/2 h-2 w-2 -translate-x-1/2 rotate-45", invalidSelection - ? "border-l border-t border-amber-800 bg-amber-950" - : "border-l border-t border-blue-800 bg-[#1B2334]" + ? "border-l border-t border-amber-800 bg-amber-950 light:border-amber-300 light:bg-amber-50" + : "border-l border-t border-blue-800 bg-[#1B2334] light:border-blue-300 light:bg-blue-50" )} /> </div> diff --git a/apps/webapp/app/components/query/QueryEditor.tsx b/apps/webapp/app/components/query/QueryEditor.tsx index ee230b26f7a..1b616059231 100644 --- a/apps/webapp/app/components/query/QueryEditor.tsx +++ b/apps/webapp/app/components/query/QueryEditor.tsx @@ -248,7 +248,7 @@ const QueryEditorForm = forwardRef< ); return ( - <div className="flex h-full flex-col gap-2 bg-background-deep pb-2"> + <div className="flex h-full flex-col gap-2 bg-white pb-2 dark:bg-background-deep"> <TSQLEditor defaultValue={query} onChange={setQuery} diff --git a/apps/webapp/app/components/queues/QueueControls.tsx b/apps/webapp/app/components/queues/QueueControls.tsx index a96f94da459..7817749b0cd 100644 --- a/apps/webapp/app/components/queues/QueueControls.tsx +++ b/apps/webapp/app/components/queues/QueueControls.tsx @@ -3,6 +3,7 @@ import { DialogClose } from "@radix-ui/react-dialog"; import { Form, useNavigation } from "@remix-run/react"; import type { QueueItem } from "@trigger.dev/core/v3/schemas"; import { useEffect, useState } from "react"; +import { cn } from "~/utils/cn"; import { Button, type ButtonVariant } from "~/components/primitives/Buttons"; import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog"; import { FormButtons } from "~/components/primitives/FormButtons"; @@ -61,18 +62,24 @@ export function QueuePauseResumeButton({ <Button type="button" variant={variant} + className={cn( + iconOnly && + (queue.paused + ? "system:border-transparent system:bg-success system:transition system:group-hover/button:bg-success system:group-hover/button:brightness-90" + : "system:border-transparent system:bg-warning system:transition system:group-hover/button:bg-warning system:group-hover/button:brightness-90"), + withQueueName && + (queue.paused + ? "border-success/60 text-success [&_span]:text-success hover:border-success" + : "border-warning/60 text-warning [&_span]:text-warning hover:border-warning") + )} LeadingIcon={queue.paused ? PlayIcon : PauseIcon} - leadingIconClassName={queue.paused ? "text-success" : "text-warning"} + leadingIconClassName={cn( + queue.paused ? "text-success" : "text-warning", + iconOnly && "system:text-white" + )} fullWidth={fullWidth} textAlignLeft={fullWidth} aria-label={label} - className={ - withQueueName - ? queue.paused - ? "border-success/60 text-success [&_span]:text-success hover:border-success" - : "border-warning/60 text-warning [&_span]:text-warning hover:border-warning" - : undefined - } > {iconOnly ? undefined diff --git a/apps/webapp/app/components/queues/QueueMetricCards.tsx b/apps/webapp/app/components/queues/QueueMetricCards.tsx index 65865205c55..294a94b8650 100644 --- a/apps/webapp/app/components/queues/QueueMetricCards.tsx +++ b/apps/webapp/app/components/queues/QueueMetricCards.tsx @@ -24,9 +24,9 @@ import { formatNumberCompact } from "~/utils/numberFormatter"; // so pages render instantly; loaders only supply live counts and identifiers. export const QUEUE_METRIC_COLORS = { - running: "var(--color-queues)", - limit: "#4D525B", - queued: "var(--color-queues)", + running: "var(--color-queues-chart)", + limit: "var(--color-queues-chart-ref)", + queued: "var(--color-queues-chart)", p50: "#22D3EE", p95: "#F59E0B", p99: "#EF4444", diff --git a/apps/webapp/app/components/runs/v3/AIFilterInput.tsx b/apps/webapp/app/components/runs/v3/AIFilterInput.tsx index 03b70499ed4..d6a6b32340b 100644 --- a/apps/webapp/app/components/runs/v3/AIFilterInput.tsx +++ b/apps/webapp/app/components/runs/v3/AIFilterInput.tsx @@ -206,7 +206,7 @@ function ErrorPopover({ <PopoverContent align="start" side="bottom" - className="w-(--radix-popover-trigger-width) min-w-(--radix-popover-trigger-width) max-w-(--radix-popover-trigger-width) border border-error/20 bg-[#2F1D24] px-3 py-2 text-xs text-text-dimmed" + className="w-(--radix-popover-trigger-width) min-w-(--radix-popover-trigger-width) max-w-(--radix-popover-trigger-width) border border-error/20 bg-[#2F1D24] light:bg-error/10 px-3 py-2 text-xs text-text-dimmed" > {error} </PopoverContent> diff --git a/apps/webapp/app/components/runs/v3/BatchStatus.tsx b/apps/webapp/app/components/runs/v3/BatchStatus.tsx index 17d02b23378..243c5eaac19 100644 --- a/apps/webapp/app/components/runs/v3/BatchStatus.tsx +++ b/apps/webapp/app/components/runs/v3/BatchStatus.tsx @@ -42,7 +42,12 @@ export function BatchStatusCombo({ } export function BatchStatusLabel({ status }: { status: BatchTaskRunStatus }) { - return <span className={batchStatusColor(status)}>{batchStatusTitle(status)}</span>; + // system-mono-label: System themes uncolor the label (see tailwind.css) + return ( + <span className={cn("system-mono-label", batchStatusColor(status))}> + {batchStatusTitle(status)} + </span> + ); } export function BatchStatusIcon({ diff --git a/apps/webapp/app/components/runs/v3/BulkAction.tsx b/apps/webapp/app/components/runs/v3/BulkAction.tsx index 3c25aa4409a..c472ebdaaef 100644 --- a/apps/webapp/app/components/runs/v3/BulkAction.tsx +++ b/apps/webapp/app/components/runs/v3/BulkAction.tsx @@ -121,13 +121,14 @@ export function BulkActionStatusLabel({ status: BulkActionStatus; className?: string; }) { + // system-mono-label: System themes uncolor the label (see tailwind.css) switch (status) { case "PENDING": - return <span className={cn("text-pending", className)}>In progress</span>; + return <span className={cn("system-mono-label text-pending", className)}>In progress</span>; case "COMPLETED": - return <span className={cn("text-success", className)}>Completed</span>; + return <span className={cn("system-mono-label text-success", className)}>Completed</span>; case "ABORTED": - return <span className={cn("text-error", className)}>Aborted</span>; + return <span className={cn("system-mono-label text-error", className)}>Aborted</span>; default: { assertNever(status); } diff --git a/apps/webapp/app/components/runs/v3/DeploymentStatus.tsx b/apps/webapp/app/components/runs/v3/DeploymentStatus.tsx index f4142cffce2..aae5f97ecb4 100644 --- a/apps/webapp/app/components/runs/v3/DeploymentStatus.tsx +++ b/apps/webapp/app/components/runs/v3/DeploymentStatus.tsx @@ -35,7 +35,8 @@ export function DeploymentStatusLabel({ isBuilt: boolean; }) { return ( - <span className={deploymentStatusClassNameColor(status)}> + // system-mono-label: System themes uncolor the label (see tailwind.css) + <span className={cn("system-mono-label", deploymentStatusClassNameColor(status))}> {deploymentStatusTitle(status, isBuilt)} </span> ); diff --git a/apps/webapp/app/components/runs/v3/RunFilters.tsx b/apps/webapp/app/components/runs/v3/RunFilters.tsx index f5980d9e597..1e4462577b5 100644 --- a/apps/webapp/app/components/runs/v3/RunFilters.tsx +++ b/apps/webapp/app/components/runs/v3/RunFilters.tsx @@ -1,9 +1,9 @@ import * as Ariakit from "@ariakit/react"; +import { GlobeLinesIcon } from "~/assets/icons/GlobeLinesIcon"; import { CalendarIcon, CpuChipIcon, FingerPrintIcon, - GlobeAltIcon, PlusIcon, RectangleStackIcon, Squares2X2Icon, @@ -282,7 +282,7 @@ export function filterIcon(filterKey: string): ReactNode | undefined { case "queues": return <RectangleStackIcon className="size-4" />; case "regions": - return <GlobeAltIcon className="size-4" />; + return <GlobeLinesIcon className="size-4" />; case "machines": return <MachineDefaultIcon className="size-4" />; case "versions": @@ -423,7 +423,7 @@ const filterTypes = [ { name: "tags", title: "Tags", icon: <TagIcon className="size-4" /> }, { name: "versions", title: "Versions", icon: <IconRotateClockwise2 className="size-4" /> }, { name: "queues", title: "Queues", icon: <RectangleStackIcon className="size-4" /> }, - { name: "regions", title: "Region", icon: <GlobeAltIcon className="size-4" /> }, + { name: "regions", title: "Region", icon: <GlobeLinesIcon className="size-4" /> }, { name: "machines", title: "Machines", icon: <MachineDefaultIcon className="size-4" /> }, { name: "run", title: "Run ID", icon: <FingerPrintIcon className="size-4" /> }, { name: "batch", title: "Batch ID", icon: <Squares2X2Icon className="size-4" /> }, @@ -673,7 +673,7 @@ function PermanentStatusFilter() { className="pl-1" /> ) : ( - <div className="flex h-6 items-center gap-1 rounded border border-border-bright bg-secondary pl-1 pr-2 text-xs text-text-bright transition group-hover:border-border-brighter group-hover:bg-surface-control"> + <div className="flex h-6 items-center gap-1 rounded border border-border-bright/50 shadow-xs bg-secondary pl-1 pr-2 text-xs text-text-bright transition group-hover:bg-background-raised"> <div className="grid size-4 place-items-center"> <div className="size-[75%] rounded-full border-2 border-text-bright" /> </div> @@ -852,7 +852,7 @@ function PermanentTasksFilter({ possibleTasks }: Pick<RunFiltersProps, "possible className="pl-1" /> ) : ( - <div className="flex h-6 items-center gap-1.5 rounded border border-border-bright bg-secondary pl-1 pr-2 text-xs text-text-bright transition group-hover:border-border-brighter group-hover:bg-surface-control"> + <div className="flex h-6 items-center gap-1.5 rounded border border-border-bright/50 shadow-xs bg-secondary pl-1 pr-2 text-xs text-text-bright transition group-hover:bg-background-raised"> {filterIcon("tasks")} <span>Tasks</span> </div> diff --git a/apps/webapp/app/components/runs/v3/RunTag.tsx b/apps/webapp/app/components/runs/v3/RunTag.tsx index 3d6c6751ce3..16d2333632e 100644 --- a/apps/webapp/app/components/runs/v3/RunTag.tsx +++ b/apps/webapp/app/components/runs/v3/RunTag.tsx @@ -1,5 +1,4 @@ import { useCallback, useMemo, useState } from "react"; -import tagLeftPath from "./tag-left.svg"; import { SimpleTooltip } from "~/components/primitives/Tooltip"; import { Link } from "@remix-run/react"; import { cn } from "~/utils/cn"; @@ -7,6 +6,26 @@ import { ClipboardCheckIcon, ClipboardIcon, XIcon } from "lucide-react"; type Tag = string | { key: string; value: string }; +function TagNotch() { + return ( + <svg + width="9" + height="25" + viewBox="0 0 9 25" + fill="none" + xmlns="http://www.w3.org/2000/svg" + className="block h-full w-2.25" + aria-hidden="true" + > + <path + d="M8.51694 0.5H10.5V24.5H8.51694C7.17088 24.5 5.94409 23.7281 5.36161 22.5146L1.69703 14.88C0.974863 13.3755 0.974863 11.6245 1.69703 10.12L5.3616 2.48544C5.94409 1.27194 7.17088 0.5 8.51694 0.5Z" + vectorEffect="non-scaling-stroke" + className="fill-background-bright stroke-grid-bright" + /> + </svg> + ); +} + export function RunTag({ tag, to, @@ -26,8 +45,8 @@ export function RunTag({ if (typeof tagResult === "string") { return ( <> - <img src={tagLeftPath} alt="" className="block h-full w-2.25" /> - <span className="flex items-center rounded-r-sm border-y border-r border-grid-bright bg-background-bright pr-1.5 text-text-dimmed group-hover:rounded-r-none group-hover:group-has-[[href]]:border-border-bright group-hover:group-has-[[href]]:text-charcoal-300"> + <TagNotch /> + <span className="flex items-center rounded-r-sm border-y border-r border-grid-bright bg-background-bright pr-1.5 text-text-dimmed group-hover:rounded-r-none group-hover:group-has-[[href]]:border-border-bright group-hover:group-has-[[href]]:text-text-bright"> {tag} </span> </> @@ -35,11 +54,11 @@ export function RunTag({ } else { return ( <> - <img src={tagLeftPath} alt="" className="block h-full w-2.25" /> - <span className="flex items-center border-y border-r border-grid-bright bg-background-bright pr-1.5 text-text-dimmed group-hover:group-has-[[href]]:border-border-bright group-hover:group-has-[[href]]:text-charcoal-300"> + <TagNotch /> + <span className="flex items-center border-y border-r border-grid-bright bg-background-bright pr-1.5 text-text-dimmed group-hover:group-has-[[href]]:border-border-bright group-hover:group-has-[[href]]:text-text-bright"> {tagResult.key} </span> - <span className="flex items-center whitespace-nowrap rounded-r-sm border-y border-r border-grid-bright bg-background-hover px-1.5 text-text-dimmed group-hover:rounded-r-none group-hover:group-has-[[href]]:border-border-bright group-hover:group-has-[[href]]:bg-background-raised group-hover:group-has-[[href]]:text-charcoal-300"> + <span className="flex items-center whitespace-nowrap rounded-r-sm border-y border-r border-grid-bright bg-background-hover px-1.5 text-text-dimmed group-hover:rounded-r-none group-hover:group-has-[[href]]:border-border-bright group-hover:group-has-[[href]]:bg-background-raised group-hover:group-has-[[href]]:text-text-bright"> {tagResult.value} </span> </> diff --git a/apps/webapp/app/components/runs/v3/SpanEvents.tsx b/apps/webapp/app/components/runs/v3/SpanEvents.tsx index c79a147b86f..069246c89b7 100644 --- a/apps/webapp/app/components/runs/v3/SpanEvents.tsx +++ b/apps/webapp/app/components/runs/v3/SpanEvents.tsx @@ -85,7 +85,7 @@ export function SpanEventError({ /> {enhancedException.message && ( <Callout variant="error"> - <pre className="text-wrap font-sans text-sm font-normal text-rose-200"> + <pre className="text-wrap font-sans text-sm font-normal text-rose-500 dark:text-rose-200"> {enhancedException.message} </pre> </Callout> diff --git a/apps/webapp/app/components/runs/v3/SpanTitle.tsx b/apps/webapp/app/components/runs/v3/SpanTitle.tsx index e01b1599cae..be363ca6ab3 100644 --- a/apps/webapp/app/components/runs/v3/SpanTitle.tsx +++ b/apps/webapp/app/components/runs/v3/SpanTitle.tsx @@ -231,7 +231,8 @@ export function eventBorderClassName(event: RunEvent) { function textClassNameForVariant(variant: TaskEventStyle["variant"]) { switch (variant) { case "primary": { - return "text-blue-500"; + // System themes: monochrome span titles, icons carry the color + return "text-blue-500 system:text-text-bright"; } default: { return "text-text-dimmed"; diff --git a/apps/webapp/app/components/runs/v3/TaskRunAttemptStatus.tsx b/apps/webapp/app/components/runs/v3/TaskRunAttemptStatus.tsx index 36333f6d968..6358ccecd9a 100644 --- a/apps/webapp/app/components/runs/v3/TaskRunAttemptStatus.tsx +++ b/apps/webapp/app/components/runs/v3/TaskRunAttemptStatus.tsx @@ -39,7 +39,10 @@ export function TaskRunAttemptStatusLabel({ status: ExtendedTaskAttemptStatus | null; }) { return ( - <span className={runAttemptStatusClassNameColor(status)}>{runAttemptStatusTitle(status)}</span> + // system-mono-label: System themes uncolor the label (see tailwind.css) + <span className={cn("system-mono-label", runAttemptStatusClassNameColor(status))}> + {runAttemptStatusTitle(status)} + </span> ); } diff --git a/apps/webapp/app/components/runs/v3/TaskRunStatus.tsx b/apps/webapp/app/components/runs/v3/TaskRunStatus.tsx index 9d20f5a525e..d0a4c74ffbb 100644 --- a/apps/webapp/app/components/runs/v3/TaskRunStatus.tsx +++ b/apps/webapp/app/components/runs/v3/TaskRunStatus.tsx @@ -149,7 +149,12 @@ export function TaskRunStatusReason({ } export function TaskRunStatusLabel({ status }: { status: TaskRunStatus }) { - return <span className={runStatusClassNameColor(status)}>{runStatusTitle(status)}</span>; + // system-mono-label: System themes uncolor the label (see tailwind.css) + return ( + <span className={cn("system-mono-label", runStatusClassNameColor(status))}> + {runStatusTitle(status)} + </span> + ); } export function TaskRunStatusIcon({ diff --git a/apps/webapp/app/components/runs/v3/WaitpointStatus.tsx b/apps/webapp/app/components/runs/v3/WaitpointStatus.tsx index 9879286e565..e5825088b3a 100644 --- a/apps/webapp/app/components/runs/v3/WaitpointStatus.tsx +++ b/apps/webapp/app/components/runs/v3/WaitpointStatus.tsx @@ -24,7 +24,10 @@ export function WaitpointStatusCombo({ export function WaitpointStatusLabel({ status }: { status: WaitpointTokenStatus }) { return ( - <span className={waitpointStatusClassNameColor(status)}>{waitpointStatusTitle(status)}</span> + // system-mono-label: System themes uncolor the label (see tailwind.css) + <span className={cn("system-mono-label", waitpointStatusClassNameColor(status))}> + {waitpointStatusTitle(status)} + </span> ); } diff --git a/apps/webapp/app/components/runs/v3/agent/AgentMessageView.tsx b/apps/webapp/app/components/runs/v3/agent/AgentMessageView.tsx index a82d8b9d0c6..e696f202559 100644 --- a/apps/webapp/app/components/runs/v3/agent/AgentMessageView.tsx +++ b/apps/webapp/app/components/runs/v3/agent/AgentMessageView.tsx @@ -136,7 +136,9 @@ export function renderPart(part: UIMessage["parts"][number], i: number) { return ( <div key={i} className="border-l-2 border-amber-500/40 pl-2"> <ChatBubble> - <div className="whitespace-pre-wrap text-xs italic text-amber-200/70">{p.text ?? ""}</div> + <div className="whitespace-pre-wrap text-xs italic text-amber-700 dark:text-amber-200/70"> + {p.text ?? ""} + </div> </ChatBubble> </div> ); diff --git a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx index a5115897f97..ae46cbe867c 100644 --- a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx @@ -483,7 +483,7 @@ function SubAgentContent({ parts }: { parts: any[] }) { if (partType === "reasoning" && part.text) { return ( <div key={j} className="border-l-2 border-amber-500/40 pl-2"> - <div className="whitespace-pre-wrap text-xs italic text-amber-200/70"> + <div className="whitespace-pre-wrap text-xs italic text-amber-700 dark:text-amber-200/70"> {part.text} </div> </div> diff --git a/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx b/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx index 11ef7aa8a52..775efd7c308 100644 --- a/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx @@ -128,7 +128,7 @@ function MetricRow({ bold?: boolean; }) { return ( - <div className="grid h-7 grid-cols-[1fr_auto] items-center gap-4 rounded-sm px-1.5 transition odd:bg-background-hover/40 @[28rem]:grid-cols-[8rem_1fr] hover:bg-white/4"> + <div className="grid h-7 grid-cols-[1fr_auto] items-center gap-4 rounded-sm px-1.5 transition odd:bg-background-hover/40 @[28rem]:grid-cols-[8rem_1fr] hover:bg-text-bright/4"> <span className="text-text-dimmed">{label}</span> <span className={`text-right @[28rem]:text-left ${ diff --git a/apps/webapp/app/components/runs/v3/ai/SpanMetricRow.tsx b/apps/webapp/app/components/runs/v3/ai/SpanMetricRow.tsx index a94dd1b3727..3faa5a64491 100644 --- a/apps/webapp/app/components/runs/v3/ai/SpanMetricRow.tsx +++ b/apps/webapp/app/components/runs/v3/ai/SpanMetricRow.tsx @@ -2,7 +2,7 @@ import type { ReactNode } from "react"; export function SpanMetricRow({ label, value }: { label: string; value: ReactNode }) { return ( - <div className="grid h-7 grid-cols-[1fr_auto] items-center gap-4 rounded-sm px-1.5 transition odd:bg-background-hover/40 @[28rem]:grid-cols-[8rem_1fr] hover:bg-white/4"> + <div className="grid h-7 grid-cols-[1fr_auto] items-center gap-4 rounded-sm px-1.5 transition odd:bg-background-hover/40 @[28rem]:grid-cols-[8rem_1fr] hover:bg-text-bright/4"> <span className="text-text-dimmed">{label}</span> <span className="text-right text-text-bright @[28rem]:text-left">{value}</span> </div> diff --git a/apps/webapp/app/components/sessions/v1/SessionStatus.tsx b/apps/webapp/app/components/sessions/v1/SessionStatus.tsx index 5c917b0d30f..69dfdf5092d 100644 --- a/apps/webapp/app/components/sessions/v1/SessionStatus.tsx +++ b/apps/webapp/app/components/sessions/v1/SessionStatus.tsx @@ -74,7 +74,12 @@ export function SessionStatusIcon({ } export function SessionStatusLabel({ status }: { status: SessionStatus }) { - return <span className={sessionStatusColor(status)}>{sessionStatusTitle(status)}</span>; + // system-mono-label: System themes uncolor the label (see tailwind.css) + return ( + <span className={cn("system-mono-label", sessionStatusColor(status))}> + {sessionStatusTitle(status)} + </span> + ); } export function SessionStatusCombo({ diff --git a/apps/webapp/app/hooks/useSystemThemeSync.ts b/apps/webapp/app/hooks/useSystemThemeSync.ts new file mode 100644 index 00000000000..6b2a678396f --- /dev/null +++ b/apps/webapp/app/hooks/useSystemThemeSync.ts @@ -0,0 +1,29 @@ +import { useEffect } from "react"; +import { type ThemePreference } from "~/utils/themePreference"; + +/** + * Keeps `data-theme` on <html> in sync with the preference. For `system` it + * follows the OS color scheme live; for pinned themes it writes the attribute + * explicitly - React can skip the write when its virtual DOM already matched + * the SSR fallback while the inline script had changed the real attribute. + * The single resolution rule (dark vs light) lives here and in the blocking + * inline script in root.tsx; downstream consumers react to the `data-theme` + * mutation (see useThemeColor). + */ +export function useSystemThemeSync(preference: ThemePreference) { + useEffect(() => { + if (preference !== "system") { + document.documentElement.setAttribute("data-theme", preference); + return; + } + + const media = window.matchMedia("(prefers-color-scheme: dark)"); + const apply = () => { + document.documentElement.setAttribute("data-theme", media.matches ? "dark" : "light"); + }; + + apply(); + media.addEventListener("change", apply); + return () => media.removeEventListener("change", apply); + }, [preference]); +} diff --git a/apps/webapp/app/hooks/useThemeColor.ts b/apps/webapp/app/hooks/useThemeColor.ts index d78fd39fa79..f088d16b525 100644 --- a/apps/webapp/app/hooks/useThemeColor.ts +++ b/apps/webapp/app/hooks/useThemeColor.ts @@ -1,4 +1,4 @@ -import { useState } from "react"; +import { useEffect, useState } from "react"; /** * Normalize any CSS color (hex, oklch, hsl, ...) to rgb()/rgba() by rendering @@ -17,16 +17,28 @@ function toRgb(color: string): string { } /** - * Resolve a theme CSS variable to a concrete, animatable color once on mount. + * Resolve a theme CSS variable to a concrete, animatable color. * framer-motion can't interpolate `var()` strings or oklch values, so animated - * colors must be resolved and normalized first. The fallback is used during - * SSR and should match the default dark theme (see tailwind.css). + * colors must be resolved and normalized first. Resolution happens in an + * effect so server and hydration renders both use the fallback — resolving + * during render caused hydration style mismatches. Long-lived components + * (e.g. the side menu) outlive theme switches, so re-resolve whenever + * `data-theme` flips on <html>. */ export function useThemeColor(variable: `--${string}`, fallback: string): string { - const [color] = useState(() => { - if (typeof document === "undefined") return fallback; - const value = getComputedStyle(document.documentElement).getPropertyValue(variable).trim(); - return value ? toRgb(value) : fallback; - }); + const [color, setColor] = useState(fallback); + useEffect(() => { + const resolve = () => { + const value = getComputedStyle(document.documentElement).getPropertyValue(variable).trim(); + if (value) setColor(toRgb(value)); + }; + resolve(); + const observer = new MutationObserver(resolve); + observer.observe(document.documentElement, { + attributes: true, + attributeFilter: ["data-theme"], + }); + return () => observer.disconnect(); + }, [variable]); return color; } diff --git a/apps/webapp/app/root.tsx b/apps/webapp/app/root.tsx index f6c403b8c11..d7ad416af24 100644 --- a/apps/webapp/app/root.tsx +++ b/apps/webapp/app/root.tsx @@ -1,4 +1,5 @@ import type { LinksFunction, LoaderFunctionArgs, MetaFunction } from "@remix-run/node"; +import type { CSSProperties } from "react"; import type { ShouldRevalidateFunction } from "@remix-run/react"; import { Links, Meta, Outlet, Scripts, ScrollRestoration } from "@remix-run/react"; import { type UseDataFunctionReturn, typedjson, useTypedLoaderData } from "remix-typedjson"; @@ -19,8 +20,15 @@ import { TimezoneSetter } from "./components/TimezoneSetter"; import { env } from "./env.server"; import { featuresForRequest } from "./features.server"; import { usePostHog } from "./hooks/usePostHog"; +import { useSystemThemeSync } from "./hooks/useSystemThemeSync"; import { getImpersonationState } from "./services/impersonation.server"; import { getUser } from "./services/session.server"; +import { + normalizeThemeContrast, + normalizeThemePreference, + type ThemePreference, +} from "~/utils/themePreference"; +import { cachedFlag } from "~/v3/featureFlags.server"; import { getTimezonePreference } from "./services/preferences/uiPreferences.server"; import { appEnvTitleTag } from "./utils"; @@ -71,6 +79,20 @@ export const loader = async ({ request }: LoaderFunctionArgs) => { }; const user = await getUser(request); + // Theme switching is feature-flagged; while off, everyone stays on the + // classic theme even if a preference was saved earlier. Admins always get + // the switcher so the team can dogfood before the flag flips. Cached: the + // root loader runs on every document request and client navigation. + const showThemeSwitcher = user + ? user.admin || (await cachedFlag({ key: "hasThemeSwitcher", defaultValue: false })) + : false; + // Logged-out pages (login, invites) always render the branded Classic look. + const themePreference: ThemePreference = showThemeSwitcher + ? normalizeThemePreference(user?.dashboardPreferences.theme) + : "classic"; + const themeContrast = showThemeSwitcher + ? normalizeThemeContrast(user?.dashboardPreferences.contrast) + : 0; // Display-only: while impersonating, an admin can ask to see the dashboard // the way the impersonated user sees it. Exposed from root so every route can // read it. @@ -99,6 +121,12 @@ export const loader = async ({ request }: LoaderFunctionArgs) => { triggerCliTag: env.TRIGGER_CLI_TAG, kapa, timezone, + showThemeSwitcher, + themePreference, + themeContrast, + // Consumed by ResizablePanel: the browser check must match between SSR + // and hydration, so it is derived from the request user-agent. + isFirefox: /firefox/i.test(request.headers.get("user-agent") ?? ""), }, { headers } ); @@ -117,7 +145,7 @@ export const shouldRevalidate: ShouldRevalidateFunction = (options) => { export function ErrorBoundary() { return ( <> - <html lang="en" className="h-full" data-theme="dark"> + <html lang="en" className="h-full" data-theme="classic"> <head> <meta charSet="utf-8" /> @@ -141,13 +169,37 @@ export function ErrorBoundary() { } export default function App() { - const { posthogProjectKey, posthogUiHost, kapa: _kapa } = useTypedLoaderData<typeof loader>(); + const { + posthogProjectKey, + posthogUiHost, + kapa: _kapa, + themePreference, + themeContrast, + } = useTypedLoaderData<typeof loader>(); usePostHog(posthogProjectKey, posthogUiHost); + useSystemThemeSync(themePreference); + // SSR falls back to dark for `system`; the inline script below corrects it + // before paint, and useSystemThemeSync keeps it live afterwards. + const resolvedTheme = themePreference === "system" ? "dark" : themePreference; return ( <> - <html lang="en" className="h-full" data-theme="dark"> + <html + lang="en" + className="h-full" + // The pre-paint script below may flip data-theme before hydration + suppressHydrationWarning + data-theme={resolvedTheme} + data-theme-preference={themePreference} + // Contrast overlay input for the System themes; Classic never reads it + style={{ "--theme-contrast": themeContrast / 100 } as CSSProperties} + > <head> + <script + dangerouslySetInnerHTML={{ + __html: `try{if(document.documentElement.getAttribute("data-theme-preference")==="system"){document.documentElement.setAttribute("data-theme",matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light")}}catch(e){}`, + }} + /> <StaleAssetRecovery isProduction={isProduction} /> <Meta /> <Links /> diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new/route.tsx index 563155468f4..f121900e901 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new/route.tsx @@ -471,7 +471,7 @@ export default function Page() { cancelButton={ <LinkButton to={v3ProjectAlertsPath(organization, project, environment)} - variant="tertiary/medium" + variant="secondary/medium" > Cancel </LinkButton> diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx index 9bf64da1603..62ecec6890f 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx @@ -1,11 +1,11 @@ import { getFormProps, useForm } from "@conform-to/react"; +import { GlobeLinesIcon } from "~/assets/icons/GlobeLinesIcon"; import { parseWithZod } from "@conform-to/zod"; import { BellAlertIcon, BellSlashIcon, BookOpenIcon, EnvelopeIcon, - GlobeAltIcon, LockClosedIcon, PlusIcon, TrashIcon, @@ -599,7 +599,7 @@ export function AlertChannelTypeIcon({ case "SLACK": return <SlackIcon className={className} />; case "WEBHOOK": - return <GlobeAltIcon className={className} />; + return <GlobeLinesIcon className={className} />; default: { assertNever(channelType); } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.apikeys/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.apikeys/route.tsx index 1d6b12bd2d0..cb6df265573 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.apikeys/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.apikeys/route.tsx @@ -193,7 +193,7 @@ export default function Page() { )} <Accordion type="single" collapsible> - <AccordionItem value="item-1"> + <AccordionItem value="item-1" className="bg-white dark:bg-transparent"> <AccordionTrigger>How to set these environment variables</AccordionTrigger> <AccordionContent> <div className="flex flex-col gap-2"> diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx index cdbe364d88e..bd152e58033 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx @@ -913,7 +913,7 @@ function DeleteEnvironmentVariableButton({ fullWidth textAlignLeft LeadingIcon={TrashIcon} - leadingIconClassName="text-rose-500 group-hover/button:text-text-bright transition-colors" + leadingIconClassName="text-rose-500 group-hover/button:text-white dark:group-hover/button:text-text-bright transition-colors" className="ml-0.5 transition-colors group-hover/button:bg-error" > {isLoading ? "Deleting" : ""} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx index ea6b33eedae..6ec1aa7fce1 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx @@ -589,7 +589,11 @@ function ErrorDetailSidebar({ <Property.Label>Error status</Property.Label> <Property.Value> <div className="flex items-center justify-between"> - <ErrorStatusBadge status={errorGroup.state.status} className="w-fit" /> + <ErrorStatusBadge + status={errorGroup.state.status} + prominence="bright" + className="w-fit" + /> <ErrorStatusDropdown state={errorGroup.state} taskIdentifier={errorGroup.taskIdentifier} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.limits/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.limits/route.tsx index 7ace6919a55..40eb8356f73 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.limits/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.limits/route.tsx @@ -12,7 +12,6 @@ import { Feedback } from "~/components/Feedback"; import { PageBody, PageContainer } from "~/components/layout/AppLayout"; import { EnvironmentSelector } from "~/components/navigation/EnvironmentSelector"; import { AnimatedNumber } from "~/components/primitives/AnimatedNumber"; -import { Badge } from "~/components/primitives/Badge"; import { Button, LinkButton } from "~/components/primitives/Buttons"; import { Header2 } from "~/components/primitives/Headers"; import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader"; @@ -198,7 +197,7 @@ function CurrentPlanSection({ return ( <div className="flex flex-col gap-3"> <Header2 className="flex items-center gap-1"> - <IconCardsFilled className="size-5 text-amber-400" /> + <IconCardsFilled className="size-5 text-amber-400 system-mono-icon" /> Current plan </Header2> <Table variant="bright/no-hover"> @@ -233,7 +232,7 @@ function ConcurrencySection({ concurrencyPath }: { concurrencyPath: string }) { return ( <div className="flex flex-col gap-3"> <Header2 className="flex items-center gap-1.5"> - <ConcurrencyIcon className="size-5 text-orange-500" /> + <ConcurrencyIcon className="size-5 text-orange-500 system-mono-icon" /> Concurrency limits <InfoIconTooltip content="Concurrency limits control how many runs execute at the same time." @@ -277,7 +276,7 @@ function RateLimitsSection({ <div className="flex flex-col gap-3"> <div className="flex items-center justify-between"> <Header2 className="flex items-center gap-1.5"> - <Gauge className="size-5 text-indigo-500" /> + <Gauge className="size-5 text-indigo-500 system-mono-icon" /> Rate limits <InfoIconTooltip content="Rate limits control how many API requests can be made within a time window. They are tracked per API key." @@ -448,21 +447,24 @@ function RateLimitTypeBadge({ switch (rateLimitType) { case "tokenBucket": return ( - <Badge variant="small" className="w-fit bg-blue-500/20 text-blue-400"> - Token bucket - </Badge> + <RateLimitTypePill + className="bg-blue-500/10 text-blue-600 dark:text-blue-400 system:text-blue-500" + label="Token bucket" + /> ); case "fixedWindow": return ( - <Badge variant="small" className="w-fit bg-purple-500/20 text-purple-400"> - Fixed window - </Badge> + <RateLimitTypePill + className="bg-purple-500/10 text-purple-600 dark:text-purple-400 system:text-purple-500" + label="Fixed window" + /> ); case "slidingWindow": return ( - <Badge variant="small" className="w-fit bg-green-500/20 text-green-400"> - Sliding window - </Badge> + <RateLimitTypePill + className="bg-green-500/10 text-green-700 dark:text-green-400 system:text-green-600" + label="Sliding window" + /> ); default: return null; @@ -544,7 +546,7 @@ function QuotasSection({ return ( <div className="flex flex-col gap-3"> <Header2 className="flex items-center gap-1"> - <IconTallymark4 className="size-6 text-blue-500" /> + <IconTallymark4 className="size-6 text-blue-500 system-mono-icon" /> Quotas <InfoIconTooltip content="Quotas define the maximum resources available to your organization." @@ -729,7 +731,7 @@ function FeaturesSection({ return ( <div className="flex flex-col gap-3"> <Header2 className="flex items-center gap-1.5"> - <IconDiamondFilled className="size-5 text-green-600" /> + <IconDiamondFilled className="size-5 text-green-600 system-mono-icon" /> Plan features </Header2> <Table variant="bright/no-hover"> @@ -861,19 +863,32 @@ function getUsageColorClass( } } +function RateLimitTypePill({ className, label }: { className: string; label: string }) { + return ( + <span + className={cn( + "contrast-chip inline-flex w-fit items-center rounded px-2 py-0.5 text-xs font-medium", + className + )} + > + {label} + </span> + ); +} + function SourceBadge({ source }: { source: "default" | "plan" | "override" }) { const variants: Record<typeof source, { label: string; className: string }> = { default: { label: "Default", - className: "bg-indigo-500/20 text-indigo-400", + className: "bg-indigo-500/10 text-indigo-600 dark:text-indigo-400 system:text-indigo-500", }, plan: { label: "Plan", - className: "bg-purple-500/20 text-purple-400", + className: "bg-purple-500/10 text-purple-600 dark:text-purple-400 system:text-purple-500", }, override: { label: "Override", - className: "bg-amber-500/20 text-amber-400", + className: "bg-amber-500/10 text-amber-700 dark:text-amber-400 system:text-warning", }, }; @@ -882,7 +897,7 @@ function SourceBadge({ source }: { source: "default" | "plan" | "override" }) { return ( <span className={cn( - "inline-flex items-center rounded px-1.5 py-0.5 text-xs font-medium", + "contrast-chip inline-flex items-center rounded px-2 py-0.5 text-xs font-medium", variant.className )} > diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models._index/route.tsx index 9a3a0d0500c..744c29235aa 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models._index/route.tsx @@ -67,6 +67,7 @@ import { TableRow, } from "~/components/primitives/Table"; import { TabButton, TabContainer } from "~/components/primitives/Tabs"; +import { SimpleTooltip } from "~/components/primitives/Tooltip"; import { UsageSparkline } from "~/components/primitives/UsageSparkline"; import { appliedSummary, @@ -1311,21 +1312,21 @@ function YourModelsTab({ <Table className="table-fixed"> <TableHeader> <TableRow> - <TableHeaderCell className="w-[18%]">Model</TableHeaderCell> - <TableHeaderCell className="w-[12%]">Provider</TableHeaderCell> + <TableHeaderCell className="w-[24%]">Model</TableHeaderCell> + <TableHeaderCell className="w-[10%]">Provider</TableHeaderCell> <TableHeaderCell className="w-[8%]" alignment="right"> Calls </TableHeaderCell> <TableHeaderCell className="w-[8%]" alignment="right"> Cost </TableHeaderCell> - <TableHeaderCell className="w-[10%]" alignment="right"> + <TableHeaderCell className="w-[9%]" alignment="right"> Cache savings </TableHeaderCell> - <TableHeaderCell className="w-[9%]" alignment="right"> + <TableHeaderCell className="w-[8%]" alignment="right"> Avg TTFC </TableHeaderCell> - <TableHeaderCell className="w-[11%]" alignment="right"> + <TableHeaderCell className="w-[9%]" alignment="right"> Avg tokens/sec </TableHeaderCell> <TableHeaderCell className="w-[12%]">Calls trend</TableHeaderCell> @@ -1351,7 +1352,13 @@ function YourModelsTab({ isSelected={!!catalogItem && selectedModelId === catalogItem.friendlyId} > <TableCell onClick={select} isTabbableCell={!!select}> - {displayId} + <SimpleTooltip + asChild + button={<span className="block truncate">{displayId}</span>} + content={displayId} + side="top" + disableHoverableContent + /> </TableCell> <TableCell onClick={select}> <span className="flex items-center gap-1.5"> diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug/route.tsx index a10461bdf56..cfdfb44fec7 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug/route.tsx @@ -555,7 +555,10 @@ export default function PromptDetailPage() { <span className="text-xs text-text-dimmed">v{selectedVersion.version}</span> {isCurrent && <Badge variant="extra-small">current</Badge>} {selectedVersion.labels.includes("override") && ( - <Badge variant="extra-small" className="border-amber-500/30 text-amber-400"> + <Badge + variant="extra-small" + className="border-amber-500/30 text-amber-400 system:border-transparent system:bg-warning system:text-white" + > override </Badge> )} @@ -613,7 +616,7 @@ export default function PromptDetailPage() { exit={{ opacity: 0, height: 0 }} transition={{ duration: 0.2, ease: "easeInOut" }} > - <span className="py-1.5 text-xs text-amber-300"> + <span className="py-1.5 text-xs text-amber-300 light:text-amber-800"> Override v{overrideVersion.version} is active. API calls resolve to this version instead of the deployed prompt. </span> @@ -622,20 +625,22 @@ export default function PromptDetailPage() { hasPermission={canWritePrompts} noPermissionTooltip="You don't have permission to edit prompt overrides" variant="tertiary/small" - className="border-amber-300/50 bg-amber-400/10 text-amber-300 group-hover/button:border-amber-400/60 group-hover/button:bg-amber-500/25 group-hover/button:text-amber-200" + className="border-amber-300/50 bg-amber-400/10 text-amber-300 group-hover/button:border-amber-400/60 group-hover/button:bg-amber-500/25 group-hover/button:text-amber-200 system:border-transparent system:bg-warning system:text-white system:transition system:group-hover/button:bg-warning system:group-hover/button:brightness-90 system:group-hover/button:text-white" onClick={() => setOverrideDialogOpen(true)} > - Edit + <span className="mx-auto grow self-center truncate text-text-bright system:text-white"> + Edit + </span> </PermissionButton> <PermissionButton hasPermission={canWritePrompts} noPermissionTooltip="You don't have permission to edit prompt overrides" variant="tertiary/small" - className="border-amber-300/50 bg-amber-400/10 text-amber-300 group-hover/button:border-amber-400/60 group-hover/button:bg-amber-500/25 group-hover/button:text-amber-200" + className="border-amber-300/50 bg-amber-400/10 text-amber-300 group-hover/button:border-amber-400/60 group-hover/button:bg-amber-500/25 group-hover/button:text-amber-200 system:border-warning/60 system:bg-transparent system:transition system:group-hover/button:bg-warning/10" onClick={() => fetcher.submit({ intent: "removeOverride" }, { method: "POST" })} disabled={fetcher.state !== "idle"} > - Remove + <span className="mx-auto grow self-center truncate text-text-bright">Remove</span> </PermissionButton> </div> </motion.div> @@ -934,7 +939,11 @@ function OverrideDialog({ className="-mx-3 w-auto flex-1 border-b border-t border-grid-dimmed" > {/* Editor */} - <ResizablePanel id="override-editor" min="300px" className="bg-[#121317]"> + <ResizablePanel + id="override-editor" + min="300px" + className="bg-[#121317] light:bg-editor-background" + > <TextEditor className="h-full" autoFocus @@ -2097,7 +2106,10 @@ function VersionsTab({ /> <span className="font-medium text-text-bright">v{v.version}</span> {isOverride && ( - <Badge variant="extra-small" className="border-amber-500/30 text-amber-400"> + <Badge + variant="extra-small" + className="border-amber-500/30 text-amber-400 system:border-transparent system:bg-warning system:text-white" + > override </Badge> )} @@ -2115,7 +2127,7 @@ function VersionsTab({ {(v.model || v.commitMessage) && ( <div className="flex items-center gap-1.5 truncate text-xs text-text-dimmed"> {v.model && <span>{v.model}</span>} - {v.model && v.commitMessage && <span className="text-charcoal-600">/</span>} + {v.model && v.commitMessage && <span className="text-text-dimmed/50">/</span>} {v.commitMessage && <span className="truncate">{v.commitMessage}</span>} </div> )} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/TableSchemaContent.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/TableSchemaContent.tsx index abcc2141f86..23647493b8b 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/TableSchemaContent.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/TableSchemaContent.tsx @@ -11,7 +11,7 @@ function ColumnHelpItem({ col }: { col: ColumnSchema }) { return ( <div className="pt-1"> <div className="flex items-center gap-2"> - <CopyableText value={col.name} className="text-sm text-indigo-400" /> + <CopyableText value={col.name} className="text-sm text-indigo-400 light:text-indigo-600" /> <Badge className="font-mono text-xxs">{col.type}</Badge> </div> {col.description && ( diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 0430058a7af..d128f53a894 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -552,7 +552,12 @@ function QueuesWithMetricsView() { valueClassName={cn(limitClassName, "tabular-nums")} suffix={ limitStatus === "burst" ? ( - <span className={cn(limitClassName, "flex items-center gap-1")}> + <span + className={cn( + limitClassName, + "system:text-text-dimmed flex items-center gap-1" + )} + > Including {envRunningLive - environment.concurrencyLimit} burst runs{" "} <BurstFactorTooltip environment={environment} /> </span> @@ -928,7 +933,7 @@ function QueuesWithMetricsView() { bucketStartMs={metrics?.bucketStartMs} bucketIntervalMs={metrics?.bucketIntervalMs} width={134} - color="var(--color-queues)" + color="var(--color-queues-chart)" unitLabel={{ singular: "queued", plural: "queued" }} showPeak={false} formatPeak={(v) => v.toLocaleString()} @@ -1275,9 +1280,9 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ when it's into burst capacity. </> ), - color: "var(--color-queues)", + color: "var(--color-queues-chart)", legend: [ - { color: "var(--color-queues)", label: "Saturation" }, + { color: "var(--color-queues-chart)", label: "Saturation" }, { color: "var(--color-warning)", label: "Over limit" }, ], query: `SELECT timeBucket() AS t,\n max(max_env_running) AS running,\n max(max_env_limit) AS env_limit\nFROM env_metrics\nGROUP BY t\nORDER BY t`, @@ -1298,7 +1303,7 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ id: "backlog", label: "Backlog", description: "How many runs are waiting across the environment, over time.", - color: "var(--color-queues)", + color: "var(--color-queues-chart)", query: `SELECT timeBucket() AS t,\n max(max_env_queued) AS queued\nFROM env_metrics\nGROUP BY t\nORDER BY t`, derive: (rows) => { const points = rows.map((r) => ({ @@ -1318,9 +1323,9 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ </> ), totalTooltip: "The worst p95 in the selected window.", - color: "var(--color-queues)", + color: "var(--color-queues-chart)", legend: [ - { color: "var(--color-queues)", label: "p95" }, + { color: "var(--color-queues-chart)", label: "p95" }, { color: "var(--color-warning)", label: "Over 1 min" }, ], query: SCHEDULING_DELAY_QUERY, @@ -1359,7 +1364,7 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ label: "Throttled", description: "How often runs were held back by a limit.", totalTooltip: "The share of the selected window with at least one blocked dequeue.", - color: "var(--color-queues)", + color: "var(--color-queues-chart)", legend: [{ color: "var(--color-warning)", label: "Throttled" }], query: THROTTLED_QUERY, derive: (rows) => { @@ -1614,20 +1619,26 @@ function queueHealthLabel({ paused, running, queued, limit }: QueueHealth): Queu return "Idle"; } +// Tint + colored text, sized like the error status chips (see ErrorStatusBadge). const QUEUE_HEALTH_STYLES: Record<QueueHealthLabel, string> = { - Paused: "text-warning", - "At capacity": "text-warning", - Backlogged: "text-blue-500", - Active: "text-success", - Idle: "text-text-dimmed", + Paused: "bg-warning/10 text-warning", + "At capacity": "bg-warning/10 text-warning", + Backlogged: "bg-blue-500/10 text-blue-500", + Active: "bg-success/10 text-success", + Idle: "bg-charcoal-500/10 text-text-dimmed", }; function QueueHealthBadge(health: QueueHealth) { const label = queueHealthLabel(health); return ( - <Badge variant="extra-small" className={cn("ml-auto w-fit", QUEUE_HEALTH_STYLES[label])}> + <span + className={cn( + "contrast-chip ml-auto inline-flex w-fit items-center rounded px-2 py-0.5 text-xs font-medium", + QUEUE_HEALTH_STYLES[label] + )} + > {label} - </Badge> + </span> ); } @@ -1724,7 +1735,12 @@ function ClassicQueuesView() { valueClassName={cn(limitClassName, "tabular-nums")} suffix={ limitStatus === "burst" ? ( - <span className={cn(limitClassName, "flex items-center gap-1")}> + <span + className={cn( + limitClassName, + "system:text-text-dimmed flex items-center gap-1" + )} + > Including {environment.running - environment.concurrencyLimit} burst runs{" "} <BurstFactorTooltip environment={environment} /> </span> @@ -1755,7 +1771,12 @@ function ClassicQueuesView() { valueClassName={limitClassName} suffix={ environment.burstFactor > 1 ? ( - <span className={cn(limitClassName, "flex items-center gap-1")}> + <span + className={cn( + limitClassName, + "system:text-text-dimmed flex items-center gap-1" + )} + > Burst limit {environment.burstFactor * environment.concurrencyLimit}{" "} <BurstFactorTooltip environment={environment} /> </span> diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 10f465f82c4..55494258f79 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -1208,7 +1208,7 @@ function ConcurrencyBlock({ <span className={cn( "text-xs tabular-nums", - atLimit ? "text-warning" : "text-text-dimmed" + atLimit ? "system-mono-label text-warning" : "text-text-dimmed" )} > {/* Separator so the limit and the percentage don't read as one number @@ -1219,7 +1219,7 @@ function ConcurrencyBlock({ )} </div> {limit !== null && limit > 0 && ( - <div className="h-1.5 w-full overflow-hidden rounded-full bg-charcoal-750"> + <div className="h-1.5 w-full overflow-hidden rounded-full bg-black/5 dark:bg-charcoal-750"> <div className={cn("h-full rounded-full", atLimit ? "bg-warning" : "bg-queues")} style={{ width: `${pct}%` }} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.regions/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.regions/route.tsx index f5b4e4901db..c4d28ded12a 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.regions/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.regions/route.tsx @@ -313,6 +313,7 @@ export default function Page() { <Paragraph variant="extra-small">Suggest a new region</Paragraph> </TableCell> <TableCellMenu + className="suggest-region-cell" alignment="right" isSticky visibleButtons={ diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx index 95dbe5f7926..085fa8f23ac 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx @@ -1490,7 +1490,7 @@ function TimelineView({ {(ms) => ( <motion.div className={cn( - "-ml-0.5 size-3 rounded-full border-2 border-background-bright", + "timeline-point -ml-0.5 size-3 rounded-full border-2 border-background-bright", eventBackgroundClassName(node.data) )} layoutId={disableSpansAnimations ? undefined : node.id} @@ -1744,7 +1744,7 @@ function SpanWithDuration({ <Timeline.Span {...props}> <motion.div className={cn( - "relative flex h-4 w-full min-w-0.5 items-center", + "timeline-span relative flex h-4 w-full min-w-0.5 items-center", eventBackgroundClassName(node.data), fadeLeft ? "rounded-r-sm bg-linear-to-r from-black/50 to-transparent" : "rounded-sm" )} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings._index/route.tsx index 6f37ac02dda..348658ba9ef 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings._index/route.tsx @@ -5,7 +5,6 @@ import { CheckIcon, ExclamationTriangleIcon, FolderIcon, - GlobeAltIcon, TrashIcon, } from "@heroicons/react/20/solid"; import { Form, type MetaFunction, useActionData, useNavigation, useSubmit } from "@remix-run/react"; @@ -13,6 +12,7 @@ import { json, type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { useEffect, useRef, useState } from "react"; import { redirect, typedjson, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; +import { GlobeLinesIcon } from "~/assets/icons/GlobeLinesIcon"; import { InlineCode } from "~/components/code/InlineCode"; import { MainHorizontallyCenteredContainer, @@ -515,7 +515,7 @@ function LogoForm({ onLoad={() => setFaviconError(false)} /> ) : ( - <GlobeAltIcon className="size-6 text-text-dimmed" /> + <GlobeLinesIcon className="size-6 text-indigo-500" /> )} </button> <Input @@ -681,7 +681,7 @@ function RadioDot({ active }: { active: boolean }) { } const iconTileClass = - "box-content grid size-10 shrink-0 place-items-center rounded-sm border-2 bg-charcoal-775"; + "box-content grid size-10 shrink-0 place-items-center rounded-sm border-2 bg-background-bright"; function toRecord(json: unknown): Record<string, unknown> { return json && typeof json === "object" ? (json as Record<string, unknown>) : {}; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.private-connections._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.private-connections._index/route.tsx index cbb1d6370e9..914e9ceb232 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.private-connections._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.private-connections._index/route.tsx @@ -114,17 +114,17 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { }; const STATUS_COLORS: Record<PrivateLinkConnectionStatus, string> = { - PENDING: "bg-amber-500/20 text-amber-400", - PROVISIONING: "bg-blue-500/20 text-blue-400", - ACTIVE: "bg-emerald-500/20 text-emerald-400", - ERROR: "bg-rose-500/20 text-rose-400", - DELETING: "bg-surface-control-active/20 text-text-dimmed", + PENDING: "bg-amber-500/10 text-amber-700 dark:text-amber-400 system:text-warning", + PROVISIONING: "bg-blue-500/10 text-blue-600 dark:text-blue-400 system:text-blue-500", + ACTIVE: "bg-emerald-500/10 text-emerald-700 dark:text-emerald-400 system:text-emerald-600", + ERROR: "bg-rose-500/10 text-rose-600 dark:text-rose-400 system:text-error", + DELETING: "bg-charcoal-500/10 text-text-dimmed", }; function StatusBadge({ status }: { status: PrivateLinkConnectionStatus }) { return ( <span - className={`inline-flex items-center rounded-full px-2 py-0.5 text-xs font-medium ${STATUS_COLORS[status]}`} + className={`contrast-chip inline-flex items-center rounded px-2 py-0.5 text-xs font-medium ${STATUS_COLORS[status]}`} > {status} </span> @@ -190,11 +190,6 @@ export default function Page() { > Private connection docs </LinkButton> - {hasPrivateNetworking && canAdd && ( - <LinkButton variant="primary/small" LeadingIcon={PlusIcon} to="new"> - Add Connection - </LinkButton> - )} </PageAccessories> </NavBar> <PageBody scrollable={true}> @@ -292,6 +287,14 @@ export default function Page() { </div> ))} + {canAdd && ( + <div className="flex justify-center pt-2"> + <LinkButton variant="primary/small" LeadingIcon={PlusIcon} to="new"> + Add Connection + </LinkButton> + </div> + )} + {!canAdd && ( <Paragraph variant="extra-small" className="text-text-dimmed"> Connection limit reached ({limit}). Delete an existing connection to add a new diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.sso/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.sso/route.tsx index 3d88220c4c2..e24fceb332c 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.sso/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.sso/route.tsx @@ -671,7 +671,7 @@ function DomainList({ domains }: { domains: ReadonlyArray<DomainRow> }) { <div className="flex flex-col"> <span className="font-mono text-sm text-text-bright">{d.domain}</span> {d.state === "failed" && d.verificationFailedReason && ( - <span className="mt-0.5 text-xs text-rose-400"> + <span className="mt-0.5 text-xs text-rose-500 dark:text-rose-400"> Reason: <span className="font-mono">{d.verificationFailedReason}</span> </span> )} @@ -851,7 +851,7 @@ function StatusIndicator({ label, active = true }: { label: string; active?: boo active ? "text-success" : "text-text-dimmed" )} > - <span className={cn("size-1.5 rounded-full", active ? "bg-success" : "bg-charcoal-500")} /> + <span className={cn("size-1.5 rounded-full", active ? "bg-success" : "bg-text-dimmed")} /> {label} </span> ); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.usage/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.usage/route.tsx index da2359a3057..03636f285d2 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.usage/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.usage/route.tsx @@ -170,7 +170,7 @@ export default function Page() { </p> </div> <div className="flex w-full flex-1 flex-col gap-1 pb-1"> - <div className="h-2 w-full overflow-hidden rounded-full bg-charcoal-700"> + <div className="h-2 w-full overflow-hidden rounded-full bg-background-raised"> <div className="h-full rounded-full bg-blue-500" style={{ diff --git a/apps/webapp/app/routes/_app.orgs.new/route.tsx b/apps/webapp/app/routes/_app.orgs.new/route.tsx index 3ae844dfbdb..b06d9bafd93 100644 --- a/apps/webapp/app/routes/_app.orgs.new/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.new/route.tsx @@ -1,6 +1,7 @@ import { getFormProps, getInputProps, useForm } from "@conform-to/react"; +import { GlobeLinesIcon } from "~/assets/icons/GlobeLinesIcon"; import { parseWithZod } from "@conform-to/zod"; -import { BuildingOffice2Icon, GlobeAltIcon } from "@heroicons/react/20/solid"; +import { BuildingOffice2Icon } from "@heroicons/react/20/solid"; import { RadioGroup } from "@radix-ui/react-radio-group"; import { json, redirect, type ActionFunction, type LoaderFunctionArgs } from "@remix-run/node"; import { Form, useActionData, useNavigation } from "@remix-run/react"; @@ -140,7 +141,7 @@ export default function NewOrganizationPage() { onLoad={() => setFaviconError(false)} /> ) : ( - GlobeAltIcon + GlobeLinesIcon ); return ( diff --git a/apps/webapp/app/routes/account._index/route.tsx b/apps/webapp/app/routes/account._index/route.tsx index 341141e638e..4ecccb33d68 100644 --- a/apps/webapp/app/routes/account._index/route.tsx +++ b/apps/webapp/app/routes/account._index/route.tsx @@ -1,7 +1,15 @@ import { getFormProps, getInputProps, useForm } from "@conform-to/react"; +import { useEffect, useState } from "react"; import { conformZodMessage, parseWithZod } from "@conform-to/zod"; -import { Form, type MetaFunction, useActionData } from "@remix-run/react"; -import { type ActionFunction, json } from "@remix-run/server-runtime"; +import { ComputerDesktopIcon, MoonIcon, SunIcon, SwatchIcon } from "@heroicons/react/20/solid"; +import { + Form, + type MetaFunction, + useActionData, + useFetcher, + useLoaderData, +} from "@remix-run/react"; +import { type ActionFunction, json, type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { z } from "zod"; import { UserProfilePhoto } from "~/components/UserProfilePhoto"; import { @@ -10,6 +18,8 @@ import { PageContainer, } from "~/components/layout/AppLayout"; import { Button } from "~/components/primitives/Buttons"; +import { Select, SelectItem } from "~/components/primitives/Select"; +import { Slider } from "~/components/primitives/Slider"; import { FormError } from "~/components/primitives/FormError"; import { Header2 } from "~/components/primitives/Headers"; import { Input } from "~/components/primitives/Input"; @@ -21,10 +31,50 @@ import { prisma } from "~/db.server"; import { useUser } from "~/hooks/useUser"; import { redirectWithSuccessMessage } from "~/models/message.server"; import { updateUser } from "~/models/user.server"; -import { requireUserId } from "~/services/session.server"; +import { + updateContrastPreference, + updateThemePreference, +} from "~/services/dashboardPreferences.server"; +import { + normalizeThemeContrast, + normalizeThemePreference, + type ThemePreference, +} from "~/utils/themePreference"; +import { cachedFlag } from "~/v3/featureFlags.server"; +import { requireUser, requireUserId } from "~/services/session.server"; import { emailSchema, MAX_EMAIL_LENGTH } from "~/utils/emailValidation"; import { accountPath } from "~/utils/pathBuilder"; +const THEME_LABELS: Record<ThemePreference, string> = { + classic: "Classic", + system: "System preference", + dark: "Dark", + light: "Light", +}; + +function themeLabel(value: ThemePreference) { + return THEME_LABELS[value]; +} + +function themeIcon(value: ThemePreference) { + switch (value) { + case "classic": + return <SwatchIcon className="size-4 text-text-dimmed" />; + case "system": + return <ComputerDesktopIcon className="size-4 text-text-dimmed" />; + case "dark": + // Moon glyph reads small at its natural size, so nudge it up inside a + // size-4 box to line up with the other icons. + return ( + <span className="grid size-4 place-items-center"> + <MoonIcon className="size-3 text-text-dimmed" /> + </span> + ); + case "light": + return <SunIcon className="size-4 text-text-dimmed" />; + } +} + export const meta: MetaFunction = () => { return [ { @@ -70,11 +120,42 @@ function createSchema( }); } +export async function loader({ request }: LoaderFunctionArgs) { + const user = await requireUser(request); + const showThemeSwitcher = + user.admin || (await cachedFlag({ key: "hasThemeSwitcher", defaultValue: false })); + return json({ showThemeSwitcher }); +} + export const action: ActionFunction = async ({ request }) => { const userId = await requireUserId(request); const formData = await request.formData(); + if (formData.get("action") === "update-theme") { + const user = await requireUser(request); + const showThemeSwitcher = + user.admin || (await cachedFlag({ key: "hasThemeSwitcher", defaultValue: false })); + if (!showThemeSwitcher) { + return json({ error: "Not available" }, { status: 404 }); + } + const theme = normalizeThemePreference(formData.get("theme")); + await updateThemePreference({ user, theme }); + return json({ success: true }); + } + + if (formData.get("action") === "update-contrast") { + const user = await requireUser(request); + const showThemeSwitcher = + user.admin || (await cachedFlag({ key: "hasThemeSwitcher", defaultValue: false })); + if (!showThemeSwitcher) { + return json({ error: "Not available" }, { status: 404 }); + } + const contrast = normalizeThemeContrast(formData.get("contrast")); + await updateContrastPreference({ user, contrast }); + return json({ success: true }); + } + const formSchema = createSchema({ isEmailUnique: async (email) => { const existingUser = await prisma.user.findFirst({ @@ -121,7 +202,31 @@ export const action: ActionFunction = async ({ request }) => { export default function Page() { const user = useUser(); + const { showThemeSwitcher } = useLoaderData<typeof loader>(); const lastSubmission = useActionData(); + const themeFetcher = useFetcher(); + const contrastFetcher = useFetcher(); + const pendingTheme = themeFetcher.formData?.get("theme"); + const pendingContrast = contrastFetcher.formData?.get("contrast"); + const contrast = + typeof pendingContrast === "string" + ? normalizeThemeContrast(pendingContrast) + : normalizeThemeContrast(user.dashboardPreferences.contrast); + const theme: ThemePreference = + typeof pendingTheme === "string" + ? normalizeThemePreference(pendingTheme) + : normalizeThemePreference(user.dashboardPreferences.theme); + + // Dragging previews the contrast via the CSS var before it persists; once the + // save settles, resnap the page and the thumb to the stored value so a failed + // or rejected save doesn't leave a phantom contrast level on screen. + const [contrastPreview, setContrastPreview] = useState(contrast); + useEffect(() => { + if (contrastFetcher.state === "idle") { + setContrastPreview(contrast); + document.documentElement.style.setProperty("--theme-contrast", String(contrast / 100)); + } + }, [contrastFetcher.state, contrast]); const [form, { name, email, marketingEmails }] = useForm({ id: "account", @@ -207,6 +312,73 @@ export default function Page() { </Button> </div> </Form> + {showThemeSwitcher && ( + <> + <div className="mb-3 mt-8 w-full border-b border-grid-dimmed pb-3"> + <Header2>Appearance</Header2> + </div> + <div className="flex w-full items-center justify-between gap-4"> + <Label>Interface theme</Label> + <Select<ThemePreference, ThemePreference> + aria-label="Interface theme" + value={theme} + setValue={(value) => + themeFetcher.submit( + { action: "update-theme", theme: value }, + { method: "post" } + ) + } + variant="secondary/small" + dropdownIcon + items={["classic", "system", "dark", "light"]} + text={(value) => ( + <span className="flex items-center gap-1.5"> + {themeIcon(value)} + {themeLabel(value)} + </span> + )} + className="w-44" + > + {(items) => + items.map((item) => ( + <SelectItem key={item} value={item} icon={themeIcon(item)}> + {themeLabel(item)} + </SelectItem> + )) + } + </Select> + </div> + {theme !== "classic" && ( + <div className="mt-4 flex w-full items-center justify-between gap-4"> + <Label>Contrast</Label> + <Slider + variant="settings" + className="w-44" + aria-label="Contrast" + min={0} + max={100} + step={5} + value={[contrastPreview]} + onValueChange={(values) => { + // Live preview before the preference persists + const value = values[0] ?? 0; + setContrastPreview(value); + document.documentElement.style.setProperty( + "--theme-contrast", + String(value / 100) + ); + }} + onValueCommit={(values) => + contrastFetcher.submit( + { action: "update-contrast", contrast: String(values[0] ?? 0) }, + { method: "post" } + ) + } + /> + </div> + )} + </> + )} </MainHorizontallyCenteredContainer> </PageBody> </PageContainer> diff --git a/apps/webapp/app/routes/invites.tsx b/apps/webapp/app/routes/invites.tsx index aa3b7809ca7..6cf667c8677 100644 --- a/apps/webapp/app/routes/invites.tsx +++ b/apps/webapp/app/routes/invites.tsx @@ -140,8 +140,10 @@ export default function Page() { > <div> <FormTitle - LeadingIcon={<EnvelopeIcon className="size-6 text-cyan-500" />} - className="mb-0 text-sky-500" + LeadingIcon={ + <EnvelopeIcon className="size-6 text-cyan-500 system:text-text-bright" /> + } + className="mb-0 text-sky-500 system:text-text-bright" title={simplur`You have ${invites.length} new invitation[|s]`} /> <FormError>{form.errors}</FormError> diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.github.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.github.tsx index 61a57701fc8..4840a424d8c 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.github.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.github.tsx @@ -46,7 +46,7 @@ import { SettingsRow, SettingsRowDescription, } from "~/components/primitives/SettingsLayout"; -import { SpinnerWhite } from "~/components/primitives/Spinner"; +import { Spinner, SpinnerWhite } from "~/components/primitives/Spinner"; import { Switch } from "~/components/primitives/Switch"; import { TextLink } from "~/components/primitives/TextLink"; import { @@ -610,7 +610,6 @@ export function ConnectGitHubRepoModal({ value="connect-repo" variant="primary/medium" LeadingIcon={isConnectRepositoryLoading ? SpinnerWhite : undefined} - leadingIconClassName="text-white" disabled={isConnectRepositoryLoading} > Connect repository @@ -1030,7 +1029,7 @@ export function ConnectedGitHubRepoForm({ ? undefined : "You don't have permission to manage the GitHub integration" } - LeadingIcon={isGitSettingsLoading ? SpinnerWhite : undefined} + LeadingIcon={isGitSettingsLoading ? Spinner : undefined} > Save </Button> @@ -1100,7 +1099,7 @@ export function GitHubSettingsPanel({ if (fetcher.state === "loading" && !data) { return ( <div className="flex items-center gap-2 text-text-dimmed"> - <SpinnerWhite className="size-4" /> + <Spinner color="blue" className="size-4" /> <span className="text-sm">Loading GitHub settings...</span> </div> ); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx index 4e37354c831..ed3e50f27a1 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx @@ -1,3 +1,4 @@ +import { GlobeLinesIcon } from "~/assets/icons/GlobeLinesIcon"; import { ArrowPathIcon, BookOpenIcon, @@ -8,7 +9,6 @@ import { CloudArrowDownIcon, EnvelopeIcon, ExclamationTriangleIcon, - GlobeAltIcon, KeyIcon, QueueListIcon, SignalIcon, @@ -429,7 +429,9 @@ function RunBody({ <Header2 className={cn( "overflow-x-hidden", - run.isAgentRun ? "text-agents" : run.isScheduled ? "text-schedules" : "text-blue-500" + run.isAgentRun ? "text-agents" : run.isScheduled ? "text-schedules" : "text-blue-500", + // System themes: monochrome title, the task icon keeps the color + "system:text-text-bright" )} > <span className="truncate"> @@ -720,7 +722,7 @@ function RunBody({ </div> <div> <div className="mb-1 flex items-center gap-1"> - <GlobeAltIcon className="size-4 text-blue-500" /> + <GlobeLinesIcon className="size-4 text-blue-500" /> <Header3>Scope</Header3> </div> <div className="flex flex-col gap-0.5 text-sm text-text-dimmed"> @@ -1472,7 +1474,7 @@ function RunError({ error }: { error: TaskRunError }) { <Header3 className="text-rose-500">{name}</Header3> {enhancedError.message && ( <Callout variant="error"> - <pre className="text-wrap font-sans text-sm font-normal text-rose-200 [word-break:break-word]"> + <pre className="text-wrap font-sans text-sm font-normal text-rose-500 dark:text-rose-200 [word-break:break-word]"> {enhancedError.message} </pre> </Callout> diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.vercel.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.vercel.tsx index 0959d61417c..792275a917d 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.vercel.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.vercel.tsx @@ -27,7 +27,7 @@ import { SettingsAlertRow, SettingsRow, } from "~/components/primitives/SettingsLayout"; -import { SpinnerWhite } from "~/components/primitives/Spinner"; +import { Spinner } from "~/components/primitives/Spinner"; import { redirectBackWithErrorMessage, redirectWithErrorMessage, @@ -597,7 +597,7 @@ function VercelSettingsRows({ tooltip={canManageVercel ? undefined : noPermissionTooltip} LeadingIcon={ isLoadingProjects - ? () => <SpinnerWhite className="size-4" /> + ? () => <Spinner color="blue" className="size-4" /> : () => <VercelLogo className="-mx-1 size-3.5 text-text-bright" /> } > @@ -1014,7 +1014,7 @@ function ConnectedVercelProjectForm({ ? undefined : "You don't have permission to manage the Vercel integration" } - LeadingIcon={isDisablingAutoAssign ? SpinnerWhite : undefined} + LeadingIcon={isDisablingAutoAssign ? Spinner : undefined} onClick={() => disableAutoAssignFetcher.submit( { action: "disable-auto-assign" }, @@ -1043,7 +1043,7 @@ function ConnectedVercelProjectForm({ ? undefined : "You don't have permission to manage the Vercel integration" } - LeadingIcon={isConfigLoading ? SpinnerWhite : undefined} + LeadingIcon={isConfigLoading ? Spinner : undefined} onClick={(event) => { if (shouldPromptClearOnSave) { event.preventDefault(); @@ -1157,7 +1157,7 @@ function VercelSettingsPanel({ if (fetcher.state === "loading" && !data) { return ( <div className="flex items-center gap-2 text-text-dimmed"> - <SpinnerWhite className="size-4" /> + <Spinner color="blue" className="size-4" /> <span className="text-sm">Loading Vercel settings...</span> </div> ); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.select-plan.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.select-plan.tsx index b72033098b7..cab7b8b41b2 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.select-plan.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.select-plan.tsx @@ -779,7 +779,7 @@ export function TierEnterprise() { <Feedback defaultValue="enterprise" button={ - <div className="flex h-10 w-full cursor-pointer items-center justify-center rounded border border-border-bright bg-tertiary px-8 text-base font-medium transition hover:border-border-brighter hover:bg-surface-control"> + <div className="flex h-10 w-full cursor-pointer items-center justify-center rounded border border-border-bright/50 bg-secondary px-8 text-base font-medium shadow-xs transition hover:bg-background-raised dark:border-border-bright dark:hover:border-border-brighter"> <span className="text-center text-text-bright">Contact us</span> </div> } diff --git a/apps/webapp/app/services/dashboardPreferences.server.ts b/apps/webapp/app/services/dashboardPreferences.server.ts index 9f5f84fe2e8..772ba338ac6 100644 --- a/apps/webapp/app/services/dashboardPreferences.server.ts +++ b/apps/webapp/app/services/dashboardPreferences.server.ts @@ -1,83 +1,29 @@ -import { z } from "zod"; import { $transaction, prisma } from "~/db.server"; import { logger } from "./logger.server"; import { type UserFromSession } from "./session.server"; - -const FavoritePage = z.object({ - /** Stable id, generated client-side when the page is favorited. */ - id: z.string(), - /** App-relative URL including any search params (filters, tabs). */ - url: z.string(), - /** Display label shown in the side menu; user-renamable. */ - label: z.string(), - /** Key into the favorite page icon registry. */ - icon: z.string().optional(), -}); - -export type FavoritePage = z.infer<typeof FavoritePage>; - -const SideMenuPreferences = z.object({ - isCollapsed: z.boolean().default(false), - /** Expanded side menu width in px, set by the resize handle. */ - width: z.number().optional(), - // Map for section collapsed states - keys are section identifiers - collapsedSections: z.record(z.string(), z.boolean()).optional(), - /** Organization-specific settings */ - organizations: z - .record( - z.string(), - z.object({ - orderedItems: z.record(z.string(), z.array(z.string())), - }) - ) - .optional(), - /** Pages the user favorited, in display order. */ - favorites: z.array(FavoritePage).optional(), - /** Custom top-to-bottom order of side menu sections (section ids). */ - sectionOrder: z.array(z.string()).optional(), - /** Per-item visibility overrides (item id -> hidden). Items absent fall back to their default. */ - hiddenItems: z.record(z.string(), z.boolean()).optional(), - /** Custom item order within a section (section id -> item ids). */ - sectionItemOrder: z.record(z.string(), z.array(z.string())).optional(), -}); - -export type SideMenuPreferences = z.infer<typeof SideMenuPreferences>; +import { + type DashboardPreferences, + type FavoritePage, + parseDashboardPreferences, + SideMenuPreferences, +} from "~/utils/dashboardPreferences"; + +export type { + DashboardPreferences, + FavoritePage, + SideMenuPreferences, +} from "~/utils/dashboardPreferences"; import { type SideMenuSectionId } from "~/components/navigation/sideMenuTypes"; export type { SideMenuSectionId }; -const DashboardPreferences = z.object({ - version: z.literal("1"), - currentProjectId: z.string().optional(), - projects: z.record( - z.string(), - z.object({ - currentEnvironment: z.object({ id: z.string() }), - }) - ), - sideMenu: SideMenuPreferences.optional(), -}); - -export type DashboardPreferences = z.infer<typeof DashboardPreferences>; +import { type ThemePreference } from "~/utils/themePreference"; +export { normalizeThemePreference, type ThemePreference } from "~/utils/themePreference"; export function getDashboardPreferences(data?: any | null): DashboardPreferences { - if (!data) { - return { - version: "1", - projects: {}, - }; - } - - const result = DashboardPreferences.safeParse(data); - if (!result.success) { - logger.error("Failed to parse DashboardPreferences", { data, error: result.error }); - return { - version: "1", - projects: {}, - }; - } - - return result.data; + return parseDashboardPreferences(data, (error) => { + logger.error("Failed to parse DashboardPreferences", { data, error }); + }); } /** @@ -171,6 +117,67 @@ export async function updateCurrentProjectEnvironmentId({ }); } +export async function updateThemePreference({ + user, + theme, +}: { + user: UserFromSession; + theme: ThemePreference; +}) { + if (user.isImpersonating) { + return; + } + + if (user.dashboardPreferences.theme === theme) { + return; + } + + // Narrow jsonb_set write: a full-blob update from the session snapshot can + // race with other preference writes and drop unrelated fields. + return prisma.$executeRaw` + UPDATE "User" + SET "dashboardPreferences" = jsonb_set( + COALESCE( + "dashboardPreferences", + '{"version":"1","projects":{}}'::jsonb + ), + '{theme}', + to_jsonb(${theme}::text) + ) + WHERE id = ${user.id} + `; +} + +export async function updateContrastPreference({ + user, + contrast, +}: { + user: UserFromSession; + contrast: number; +}) { + if (user.isImpersonating) { + return; + } + + if (user.dashboardPreferences.contrast === contrast) { + return; + } + + // Narrow jsonb_set write: see updateThemePreference. + return prisma.$executeRaw` + UPDATE "User" + SET "dashboardPreferences" = jsonb_set( + COALESCE( + "dashboardPreferences", + '{"version":"1","projects":{}}'::jsonb + ), + '{contrast}', + to_jsonb(${contrast}::int) + ) + WHERE id = ${user.id} + `; +} + export async function clearCurrentProject({ user }: { user: UserFromSession }) { if (user.isImpersonating) { return; diff --git a/apps/webapp/app/tailwind.css b/apps/webapp/app/tailwind.css index ab845b63353..174a9a9ff5e 100644 --- a/apps/webapp/app/tailwind.css +++ b/apps/webapp/app/tailwind.css @@ -158,6 +158,7 @@ --color-surface-control: var(--color-charcoal-600); --color-surface-control-hover: var(--color-charcoal-550); --color-surface-control-active: var(--color-charcoal-500); + --color-input-bg: var(--color-charcoal-750); /* Borders, from subtlest to most visible */ --color-grid-dimmed: var(--color-charcoal-750); @@ -223,6 +224,84 @@ --color-callout-pricing-text: var(--color-indigo-300); } +/* + Theme overrides. A theme re-maps ONLY the semantic tokens above, never the raw + palette values. That invariant is what lets a future [data-contrast] overlay + re-map semantic tokens on top of whichever theme is active. +*/ + +/* + System preference unified accents (dark+light); Classic keeps the original + set. One shared value per accent token, >=3:1 against both dark cards and + white. Accents whose Classic default already clears both modes (blue-500, + indigo-500, pink-500, purple-500, red-500, violet-500, fuchsia-500, + rose-600) are not repeated here. Text-sized tokens (text-link, callout + text) stay per-mode: no color reaches 4.5:1 on both #1a1b1f and #ffffff. + Dark derives everything else (monochrome surfaces etc.) from Classic. +*/ +:is([data-theme="dark"], [data-theme="light"]) { + /* Status */ + --color-success: var(--color-mint-600); + --color-warning: var(--color-amber-600); + /* Env set: pink dev and green prod as in Classic, preview moves to blue + (yellow's unified mid-tone reads muddy brown). Staging stays warm orange + and is the one per-mode env color - see the light block. */ + --color-prod: var(--color-mint-600); + --color-preview: var(--color-blue-500); + + /* Icons */ + --color-schedules: var(--color-yellow-700); + /* Matches the preview env color */ + --color-previewBranches: var(--color-blue-500); + --color-metrics: var(--color-green-600); + --color-regions: var(--color-green-600); + --color-aiMetrics: var(--color-green-600); + --color-bulkActions: var(--color-emerald-600); + --color-concurrency: var(--color-amber-600); + --color-errors: var(--color-amber-600); + --color-apiKeys: var(--color-amber-600); + + /* Queue charts read blue here; Classic keeps the queues purple */ + --color-queues-chart: var(--color-blue-500); + --color-queues-chart-ref: var(--color-charcoal-500); + + /* Callout accents (callout text/bg tints stay per-mode) */ + --color-callout-docs: var(--color-blue-500); + --color-callout-pending: var(--color-blue-500); + --color-callout-pricing: var(--color-indigo-500); + + /* Amber run statuses: three distinct steps, Classic's dark-to-light order + kept (waiting-for-deploy < pending-version < paused) */ + --color-run-waiting-for-deploy: var(--color-amber-700); + --color-run-pending-version: #c76508; + --color-run-paused: var(--color-amber-600); + /* Nudged under 3:1's ceiling on white; stays lighter than system-failure */ + --color-run-timed-out: #ed5f74; +} + +/* System themes drop decorative icon accents to monochrome; Classic keeps the + colored icons. side-menu-active-icon is set in SideMenuItem for the active + nav item; system-mono-icon marks section-header icons (e.g. the limits page). */ +:is([data-theme="dark"], [data-theme="light"]) :is(.side-menu-active-icon, .system-mono-icon) { + color: var(--color-text-bright); +} + +/* System themes: status/env labels follow the surrounding text color, only the + icon keeps its tint. Classic colors both. Set in EnvironmentLabel and the + status combo components. */ +:is([data-theme="dark"], [data-theme="light"]) .system-mono-label { + color: inherit; +} + +/* Tinted status chips respond to the contrast control: a ring in the chip's + own text color fades in as contrast rises, so the soft tint keeps its + footprint but the chip gains definition. Transparent at contrast 0; + Classic never sets the variable. Marker set by the chip components. */ +:is([data-theme="dark"], [data-theme="light"]) .contrast-chip { + box-shadow: inset 0 0 0 1px + color-mix(in srgb, currentcolor calc(var(--theme-contrast, 0) * 70%), transparent); +} + /* Code syntax palette - the trigger-dark highlight theme, shared by the shiki theme (streamdown) and the prism theme (CodeBlock). Consumed from JS via @@ -290,6 +369,14 @@ --color-editor-scrollbar-thumb-active: #3c4b62; } +/* Queue chart series colors - queues purple in Classic, overridden to blue in + the System themes; the grey reference series lightens per mode there. + Consumed from JS via var(), so declared static. */ +@theme static { + --color-queues-chart: var(--color-queues); + --color-queues-chart-ref: #4d525b; +} + /* Run-status chart colors. In-between shades are intentional - statuses within a family (blues, roses, charcoals) are evenly spaced so chart series stay @@ -333,8 +420,17 @@ @custom-variant lg-height (@media (max-height: 750px)); @custom-variant md-height (@media (max-height: 600px)); -/* dark: follows the app theme (data-theme on <html>), not the OS preference */ -@custom-variant dark (&:where([data-theme="dark"], [data-theme="dark"] *)); +/* dark: follows the app theme (data-theme on <html>), not the OS preference. + Classic and Dark are both dark-mode themes, so the variant matches both. */ +@custom-variant dark (&:where([data-theme="dark"], [data-theme="classic"], [data-theme="dark"] *, [data-theme="classic"] *)); + +/* system: matches the System preference themes (Dark and Light) but never + Classic - for restyles that must leave Classic untouched. */ +@custom-variant system (&:where([data-theme="dark"], [data-theme="light"], [data-theme="dark"] *, [data-theme="light"] *)); + +/* light: the Light theme only - for values that are fine on every dark theme + but illegible on white. Classic is never matched. */ +@custom-variant light (&:where([data-theme="light"], [data-theme="light"] *)); @utility focus-custom { &:focus-visible { @@ -489,7 +585,8 @@ --chart-5: 27 87% 67%; } - [data-theme="dark"] { + /* Classic and Dark share these HSL chart vars - they were the app-wide dark values. */ + :is([data-theme="dark"], [data-theme="classic"]) { --chart-1: 220 70% 50%; --chart-2: 160 60% 45%; --chart-3: 30 80% 55%; @@ -497,6 +594,16 @@ --chart-5: 340 75% 55%; } + /* System preference unified chart set (dark+light): Classic's hues, with + chart-2/3 darkened to clear 3:1 on white. Classic keeps the set above. */ + :is([data-theme="dark"], [data-theme="light"]) { + --chart-1: 220 70% 50%; + --chart-2: 160 60% 40%; + --chart-3: 30 80% 47%; + --chart-4: 280 65% 60%; + --chart-5: 340 75% 55%; + } + /* Override react-grid-layout placeholder color (default is red) */ .react-grid-item.react-grid-placeholder { background: rgb(99 102 241) !important; /* indigo-500 */ @@ -640,6 +747,239 @@ } } +/* + Light theme. Overrides the themable variables only; raw palettes stay put. + Code/editor values come from the trigger.light VS Code theme. +*/ +[data-theme="light"] { + --color-secondary: #ffffff; + --color-input-bg: #ffffff; + + /* shadcn vars consumed by charts/streamdown (tooltip cursor fill etc.) */ + --background: #ffffff; + --foreground: #1a1b1f; + --muted: #eceef1; + --muted-foreground: #5f6570; + --border: #e2e4e9; + --sidebar: #f6f7f8; + --primary-foreground: #ffffff; + + /* Text */ + --color-primary: var(--color-apple-600); + --color-tertiary: #eef0f3; + --color-text-link: var(--color-lavender-600); + --color-text-faint: var(--color-charcoal-400); + --color-text-dimmed: var(--color-charcoal-500); + --color-text-bright: var(--color-charcoal-800); + + /* Surfaces */ + --color-background-deep: #f1f2f4; + --color-background-dimmed: #fbfbfc; + --color-background-bright: #ffffff; + --color-background-hover: #f2f3f5; + --color-background-raised: #e9eaee; + --color-surface-control: #dcdee3; + --color-surface-control-hover: #cfd2d9; + --color-surface-control-active: #b8bcc6; + + /* Borders */ + --color-grid-dimmed: #eceef1; + --color-grid-bright: #e2e4e9; + --color-border-bright: #d2d5db; + --color-border-brighter: #b9bdc7; + --color-border-brightest: #9ba1ad; + + /* Status/env/icon accents live in the unified dark+light block above; + dev keeps its Classic pink-500, which passes on white too. */ + + /* Staging is the one per-mode env color: warm orange has no clean unified + mid-tone, so dark keeps Classic's bright orange-400 and light deepens it */ + --color-staging: var(--color-orange-600); + + /* Grey chart reference series, light enough not to dominate on white */ + --color-queues-chart-ref: #c3c7cf; + + /* Neutral run statuses: soft light grays for sparkbars and charts */ + --color-run-pending: #ccd0d6; + --color-run-delayed: #d3d6db; + --color-run-waiting-to-resume: #c5c9d0; + --color-run-canceled: #d8dbdf; + --color-run-expired: #dee0e4; + + /* Monochrome icon gray - per-mode, not part of the unified accent set */ + --color-customDashboards: var(--color-charcoal-500); + + /* Callout text/bg - deep tints instead of the dark theme's pastels; text + can't unify across modes (see the unified accents block) */ + --color-callout-warning-text: var(--color-yellow-800); + --color-callout-error-text: var(--color-rose-700); + --color-callout-success-text: var(--color-green-800); + --color-callout-docs-text: var(--color-blue-800); + --color-callout-pending-bg: var(--color-blue-100); + --color-callout-pending-text: var(--color-blue-800); + --color-callout-pricing-bg: var(--color-indigo-100); + --color-callout-pricing-text: var(--color-indigo-800); + + /* Code syntax - trigger.light */ + --color-code-background: #ffffff; + --color-code-foreground: #333333; + --color-code-line-number: #a8a8ad; + --color-code-plain: #2e2e4b; + --color-code-muted: #333333; + --color-code-comment: #767a81; + --color-code-keyword: #b114d3; + --color-code-storage: #b114d3; + --color-code-type: #b114d3; + --color-code-function: #6532f5; + --color-code-variable: #404040; + --color-code-constant: #1e1e1e; + --color-code-language: #b114d3; + --color-code-object-key: #222222; + --color-code-string: #262626; + --color-code-template-punctuation: #0879e2; + --color-code-number: #262626; + --color-code-builtin: #3080e0; + --color-code-attribute: #222222; + --color-code-escape: #222222; + --color-code-regexp: #dc3545; + --color-code-regexp-constant: #5f6570; + --color-code-invalid: #dc3545; + --color-code-deleted: #dc3545; + --color-code-jsx-text: #2e2e4b; + + /* CodeMirror - trigger.light */ + --color-editor-background: #ffffff; + --color-editor-foreground: #2e2e4b; + --color-editor-keyword: #b114d3; + --color-editor-name: #197c3c; + --color-editor-function: #6532f5; + --color-editor-constant: #3080e0; + --color-editor-type: #2980b9; + --color-editor-operator: #333333; + --color-editor-comment: #767a81; + --color-editor-heading: #2c3e50; + --color-editor-string: #262626; + --color-editor-invalid: #dc3545; + --color-editor-panel-background: #f4f4f6; + --color-editor-highlight-background: #f8f8fa; + --color-editor-tooltip-background: #ffffff; + --color-editor-selection: #d9dce3; + --color-editor-cursor: #2e2e4b; + --color-editor-search-match: #0879e226; + --color-editor-search-match-outline: #0879e2; + --color-editor-search-match-selected: #0879e240; + --color-editor-selection-match: #e8e8ed; + --color-editor-matching-bracket: rgba(240, 241, 244, 0.9); + --color-editor-matching-bracket-outline: rgba(160, 166, 180, 0.5); + --color-editor-fold-placeholder: #555555; + --color-editor-scrollbar-track-active: #e8e9ec; + --color-editor-scrollbar-thumb: #c9ccd4; + --color-editor-scrollbar-thumb-active: #aeb3be; +} + +/* Streamdown's muted surface has no semantic token (charcoal-775); theme it here */ +[data-theme="light"] .streamdown-container { + --muted: #eceef1; +} + +/* The timeline label shadow is a Classic legibility aid; the System themes + drop it entirely */ +:is([data-theme="dark"], [data-theme="light"]) .text-shadow-custom { + text-shadow: none; +} + +/* Neutral timeline points: invert to a light dot with a gray ring */ +[data-theme="light"] .timeline-point.bg-surface-control-active { + border-color: var(--color-surface-control-active); + background-color: var(--color-background-bright); +} + +/* Run timeline bars: no fade gradient on light */ +[data-theme="light"] .timeline-span { + background-image: none; +} +/* On saturated bars the duration label keeps the dark-theme treatment, + but only when the bar is wide enough to contain the label — on narrow + bars the sticky label overflows onto the page background, where the + default dark-on-light text is correct. */ +[data-theme="light"] .timeline-span.bg-success, +[data-theme="light"] .timeline-span.bg-error, +[data-theme="light"] .timeline-span.bg-blue-500 { + container-type: inline-size; +} +@container (min-width: 3.5rem) { + [data-theme="light"] .timeline-span.bg-success .text-shadow-custom, + [data-theme="light"] .timeline-span.bg-error .text-shadow-custom, + [data-theme="light"] .timeline-span.bg-blue-500 .text-shadow-custom { + color: #ffffff; + } +} + +/* The table row-hover menu wraps the always-visible "Suggest a region" button + in a ring container; on light that ring doubles up with the button's own + border, so drop it here. */ +[data-theme="light"] .suggest-region-cell > div > div { + box-shadow: none; +} + +/* + Interface contrast (System themes only). --theme-contrast is 0..1, set on + <html> from the dashboard preference. It pulls the whole monochrome scale + apart: text mixes toward the mode's foreground, backgrounds toward its + depth, borders step up. Accents are untouched. srgb mixing keeps contrast 0 + byte-identical to the base values, and Classic never reads the variable. + These blocks must stay below the [data-theme="light"] theme block so they + win the cascade. +*/ +[data-theme="dark"] { + /* Text brightens toward white */ + --color-text-bright: color-mix(in srgb, var(--color-charcoal-200), #fff calc(var(--theme-contrast, 0) * 100%)); + --color-text-dimmed: color-mix(in srgb, var(--color-charcoal-400), #fff calc(var(--theme-contrast, 0) * 60%)); + --color-text-faint: color-mix(in srgb, var(--color-charcoal-500), #fff calc(var(--theme-contrast, 0) * 45%)); + + /* Backgrounds deepen toward black, keeping their relative order */ + --color-background-deep: color-mix(in srgb, var(--color-charcoal-900), #000 calc(var(--theme-contrast, 0) * 60%)); + --color-background-dimmed: color-mix(in srgb, var(--color-charcoal-850), #000 calc(var(--theme-contrast, 0) * 55%)); + --color-background-bright: color-mix(in srgb, var(--color-charcoal-800), #000 calc(var(--theme-contrast, 0) * 45%)); + --color-background-hover: color-mix(in srgb, var(--color-charcoal-750), #000 calc(var(--theme-contrast, 0) * 35%)); + --color-background-raised: color-mix(in srgb, var(--color-charcoal-700), #000 calc(var(--theme-contrast, 0) * 25%)); + --color-input-bg: color-mix(in srgb, var(--color-charcoal-750), #000 calc(var(--theme-contrast, 0) * 35%)); + + /* Controls and borders step up toward white */ + --color-surface-control: color-mix(in srgb, var(--color-charcoal-600), #fff calc(var(--theme-contrast, 0) * 12%)); + --color-surface-control-hover: color-mix(in srgb, var(--color-charcoal-550), #fff calc(var(--theme-contrast, 0) * 14%)); + --color-surface-control-active: color-mix(in srgb, var(--color-charcoal-500), #fff calc(var(--theme-contrast, 0) * 16%)); + --color-grid-dimmed: color-mix(in srgb, var(--color-charcoal-750), #fff calc(var(--theme-contrast, 0) * 18%)); + --color-grid-bright: color-mix(in srgb, var(--color-charcoal-700), #fff calc(var(--theme-contrast, 0) * 20%)); + --color-border-bright: color-mix(in srgb, var(--color-charcoal-600), #fff calc(var(--theme-contrast, 0) * 24%)); + --color-border-brighter: color-mix(in srgb, var(--color-charcoal-550), #fff calc(var(--theme-contrast, 0) * 27%)); + --color-border-brightest: color-mix(in srgb, var(--color-charcoal-500), #fff calc(var(--theme-contrast, 0) * 30%)); +} + +[data-theme="light"] { + /* Text deepens toward black */ + --color-text-bright: color-mix(in srgb, #1a1b1f, #000 calc(var(--theme-contrast, 0) * 100%)); + --color-text-dimmed: color-mix(in srgb, var(--color-charcoal-500), #000 calc(var(--theme-contrast, 0) * 85%)); + --color-text-faint: color-mix(in srgb, var(--color-charcoal-400), #000 calc(var(--theme-contrast, 0) * 70%)); + + /* On white the foregrounds carry the contrast: cards stay white while the + page-behind surfaces darken a touch so panels separate */ + --color-background-deep: color-mix(in srgb, #f1f2f4, #000 calc(var(--theme-contrast, 0) * 10%)); + --color-background-hover: color-mix(in srgb, #f2f3f5, #000 calc(var(--theme-contrast, 0) * 8%)); + --color-background-raised: color-mix(in srgb, #e9eaee, #000 calc(var(--theme-contrast, 0) * 10%)); + + /* Controls and borders push hard toward black - this is where light-mode + contrast is actually visible */ + --color-surface-control: color-mix(in srgb, #dcdee3, #000 calc(var(--theme-contrast, 0) * 25%)); + --color-surface-control-hover: color-mix(in srgb, #cfd2d9, #000 calc(var(--theme-contrast, 0) * 28%)); + --color-surface-control-active: color-mix(in srgb, #b8bcc6, #000 calc(var(--theme-contrast, 0) * 32%)); + --color-grid-dimmed: color-mix(in srgb, #eceef1, #000 calc(var(--theme-contrast, 0) * 28%)); + --color-grid-bright: color-mix(in srgb, #e2e4e9, #000 calc(var(--theme-contrast, 0) * 32%)); + --color-border-bright: color-mix(in srgb, #d2d5db, #000 calc(var(--theme-contrast, 0) * 38%)); + --color-border-brighter: color-mix(in srgb, #b9bdc7, #000 calc(var(--theme-contrast, 0) * 42%)); + --color-border-brightest: color-mix(in srgb, #9ba1ad, #000 calc(var(--theme-contrast, 0) * 46%)); +} + form:has(.unlock-hint-staging-env:hover) [data-unlock-target="staging-env"], form:has(.unlock-hint-pull-prod:hover) [data-unlock-target="pull-prod"], form:has(.unlock-hint-pull-stg:hover) [data-unlock-target="pull-stg"], diff --git a/apps/webapp/app/utils/dashboardPreferences.ts b/apps/webapp/app/utils/dashboardPreferences.ts new file mode 100644 index 00000000000..ea0c6d8bfe2 --- /dev/null +++ b/apps/webapp/app/utils/dashboardPreferences.ts @@ -0,0 +1,92 @@ +import { z } from "zod"; +import { ThemePreference } from "~/utils/themePreference"; + +/* Schema and pure parsing for the User.dashboardPreferences JSON column. + Kept out of the .server module so tests can exercise the schema without + pulling in the server env graph. */ + +export const FavoritePage = z.object({ + /** Stable id, generated client-side when the page is favorited. */ + id: z.string(), + /** App-relative URL including any search params (filters, tabs). */ + url: z.string(), + /** Display label shown in the side menu; user-renamable. */ + label: z.string(), + /** Key into the favorite page icon registry. */ + icon: z.string().optional(), +}); + +export type FavoritePage = z.infer<typeof FavoritePage>; + +export const SideMenuPreferences = z.object({ + isCollapsed: z.boolean().default(false), + /** Expanded side menu width in px, set by the resize handle. */ + width: z.number().optional(), + // Map for section collapsed states - keys are section identifiers + collapsedSections: z.record(z.string(), z.boolean()).optional(), + /** Organization-specific settings */ + organizations: z + .record( + z.string(), + z.object({ + orderedItems: z.record(z.string(), z.array(z.string())), + }) + ) + .optional(), + /** Pages the user favorited, in display order. */ + favorites: z.array(FavoritePage).optional(), + /** Custom top-to-bottom order of side menu sections (section ids). */ + sectionOrder: z.array(z.string()).optional(), + /** Per-item visibility overrides (item id -> hidden). Items absent fall back to their default. */ + hiddenItems: z.record(z.string(), z.boolean()).optional(), + /** Custom item order within a section (section id -> item ids). */ + sectionItemOrder: z.record(z.string(), z.array(z.string())).optional(), +}); + +export type SideMenuPreferences = z.infer<typeof SideMenuPreferences>; + +const DashboardPreferences = z.object({ + version: z.literal("1"), + /* An unknown value (e.g. written by a newer deploy) degrades to undefined + instead of failing the whole blob and erasing every other setting */ + theme: ThemePreference.optional().catch(undefined), + /** Interface contrast for the System themes, 0-100. */ + contrast: z.number().int().min(0).max(100).optional().catch(undefined), + currentProjectId: z.string().optional(), + projects: z.record( + z.string(), + z.object({ + currentEnvironment: z.object({ id: z.string() }), + }) + ), + sideMenu: SideMenuPreferences.optional(), +}); + +export type DashboardPreferences = z.infer<typeof DashboardPreferences>; + +/* A function, not a shared constant: the writers mutate through these objects, + so each caller needs its own container */ +function defaultPreferences(): DashboardPreferences { + return { + version: "1", + projects: {}, + }; +} + +/** Parses the stored JSON, falling back to defaults on missing or invalid data. */ +export function parseDashboardPreferences( + data?: any | null, + onError?: (error: z.ZodError) => void +): DashboardPreferences { + if (!data) { + return defaultPreferences(); + } + + const result = DashboardPreferences.safeParse(data); + if (!result.success) { + onError?.(result.error); + return defaultPreferences(); + } + + return result.data; +} diff --git a/apps/webapp/app/utils/themePreference.ts b/apps/webapp/app/utils/themePreference.ts new file mode 100644 index 00000000000..2b6408c2abc --- /dev/null +++ b/apps/webapp/app/utils/themePreference.ts @@ -0,0 +1,25 @@ +import { z } from "zod"; + +// Shared between server (dashboard preferences) and client (theme UI, system +// theme sync) - must stay free of server-only imports. +export const ThemePreference = z.enum(["classic", "system", "dark", "light"]); +export type ThemePreference = z.infer<typeof ThemePreference>; + +/** Coerce any stored/legacy value into a valid preference. Missing or unknown + * values fall back to `dark` - the new dark theme is the default (pinned, not + * system-resolved, so nobody gets surprised by light mode). */ +export function normalizeThemePreference(value: unknown): ThemePreference { + const result = ThemePreference.safeParse(value); + return result.success ? result.data : "dark"; +} + +/** The default dark theme ships with a slight contrast bump. */ +export const DEFAULT_THEME_CONTRAST = 50; + +/** Interface contrast for the System themes, 0 to 100. Missing or invalid + * values fall back to the default bump. */ +export function normalizeThemeContrast(value: unknown): number { + const num = typeof value === "string" ? Number(value) : value; + if (typeof num !== "number" || !Number.isFinite(num)) return DEFAULT_THEME_CONTRAST; + return Math.min(100, Math.max(0, Math.round(num))); +} diff --git a/apps/webapp/app/v3/featureFlags.server.ts b/apps/webapp/app/v3/featureFlags.server.ts index a813bc13b5d..c79bd8c01d7 100644 --- a/apps/webapp/app/v3/featureFlags.server.ts +++ b/apps/webapp/app/v3/featureFlags.server.ts @@ -55,6 +55,32 @@ export function makeFlag(_prisma: PrismaClientOrTransaction = prisma) { return flag; } +const cachedFlagStore = new Map<string, { value: unknown; expiresAt: number }>(); + +/** + * flag() behind a short process-level TTL cache, for global flags read on hot + * paths (e.g. the root loader) where a database round-trip per request is too + * expensive. Flips propagate within ttlMs per process. Overrides are rejected + * by the type: a scoped resolution must never be reused across scopes. + */ +export async function cachedFlag<T extends FeatureFlagKey>( + opts: Omit<FlagsOptions<T>, "overrides"> & { + defaultValue: z.infer<(typeof FeatureFlagCatalog)[T]>; + }, + ttlMs = 30_000 +): Promise<z.infer<(typeof FeatureFlagCatalog)[T]>> { + // defaultValue resolves the flag when the row is absent, so it's part of the key + const cacheKey = `${opts.key}:${JSON.stringify(opts.defaultValue)}`; + const hit = cachedFlagStore.get(cacheKey); + if (hit && hit.expiresAt > Date.now()) { + return hit.value as z.infer<(typeof FeatureFlagCatalog)[T]>; + } + + const value = await flag(opts); + cachedFlagStore.set(cacheKey, { value, expiresAt: Date.now() + ttlMs }); + return value; +} + export function makeSetFlag(_prisma: PrismaClientOrTransaction = prisma) { return async function setFlag<T extends FeatureFlagKey>( opts: FlagsOptions<T> & { value: z.infer<(typeof FeatureFlagCatalog)[T]> } diff --git a/apps/webapp/app/v3/featureFlags.ts b/apps/webapp/app/v3/featureFlags.ts index 74074214812..f617cab16f3 100644 --- a/apps/webapp/app/v3/featureFlags.ts +++ b/apps/webapp/app/v3/featureFlags.ts @@ -10,6 +10,7 @@ export const FEATURE_FLAG = { hasComputeAccess: "hasComputeAccess", hasPrivateConnections: "hasPrivateConnections", hasSso: "hasSso", + hasThemeSwitcher: "hasThemeSwitcher", mollifierEnabled: "mollifierEnabled", workerQueueScheduledSplitEnabled: "workerQueueScheduledSplitEnabled", internalApiOriginEnabled: "internalApiOriginEnabled", @@ -41,6 +42,8 @@ export const FeatureFlagCatalog = { [FEATURE_FLAG.hasComputeAccess]: z.coerce.boolean(), [FEATURE_FLAG.hasPrivateConnections]: z.coerce.boolean(), [FEATURE_FLAG.hasSso]: z.coerce.boolean(), + // Gates the Interface theme setting in /account. Off by default. + [FEATURE_FLAG.hasThemeSwitcher]: z.coerce.boolean(), [FEATURE_FLAG.mollifierEnabled]: z.coerce.boolean(), [FEATURE_FLAG.workerQueueScheduledSplitEnabled]: z.coerce.boolean(), // Routes deployed runs' TRIGGER_API_URL to INTERNAL_API_ORIGIN. Per-org, with diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 42900ea2f55..90dc46ac088 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -19,6 +19,7 @@ "guard:runops-legacy": "cross-env NODE_OPTIONS=\"--max-old-space-size=8192\" tsx ./scripts/runOpsLegacyGuard.ts", "db:seed": "tsx seed.ts", "db:seed:ai-spans": "tsx seed-ai-spans.mts", + "db:seed:queue-metrics": "tsx seed-queue-metrics.mts", "upload:sourcemaps": "bash ./upload-sourcemaps.sh", "test": "vitest --no-file-parallelism", "eval:dev": "evalite watch" diff --git a/apps/webapp/seed-queue-metrics.mts b/apps/webapp/seed-queue-metrics.mts new file mode 100644 index 00000000000..911ce51d9c6 --- /dev/null +++ b/apps/webapp/seed-queue-metrics.mts @@ -0,0 +1,991 @@ +import { prisma } from "./app/db.server"; +import { createOrganization } from "./app/models/organization.server"; +import { createProject } from "./app/models/project.server"; +import { ClickHouse } from "@internal/clickhouse"; +import type { QueueMetricsRawV1Input } from "@internal/clickhouse"; +import { generateFriendlyId } from "./app/v3/friendlyIdentifiers"; + +// Queue metrics simulator: writes realistic raw rows into a synthetic tenant's +// queue_metrics_raw_v1 and lets the MV build queue_metrics_v1 (the same path the real +// consumer uses), so the dashboard can be built without the run engine. See TRI-10407. + +const ORG_TITLE = "Queue Metrics Dev"; +const PROJECT_NAME = "queue-metrics-demo"; + +type Rng = () => number; +type QueueProfile = { + name: string; + limit: (bucket: number) => number; + arrivals: (bucket: number, rng: Rng) => number; // expected new runs enqueued this bucket + waitBaseMs: number; + sparse?: boolean; // emit no rows when the queue is fully idle (tests carry-forward gaps) + // Concurrency-key queue: adds CK-health gauge fields + live ckIndex staging (--usage) + ck?: { + backlogged: (bucket: number, rng: Rng) => number; + maxWaitMs: (bucket: number, rng: Rng) => number; + }; +}; +type Scenario = { + description: string; + envLimit: (bucket: number) => number; + queues: QueueProfile[]; +}; + +// --------------------------------------------------------------------------- +// CLI args +// --------------------------------------------------------------------------- + +function parseArgs(argv: string[]) { + const flags: Record<string, string> = {}; + for (let i = 0; i < argv.length; i++) { + const t = argv[i]; + if (t.startsWith("--")) { + const k = t.slice(2); + const n = argv[i + 1]; + if (n && !n.startsWith("--")) { + flags[k] = n; + i++; + } else flags[k] = "true"; + } + } + return flags; +} + +function parseDuration(s: string): number { + const m = s.match(/^(\d+)\s*(s|m|h|d)?$/); + if (!m) throw new Error(`bad duration: ${s}`); + const n = Number(m[1]); + const unit = m[2] ?? "s"; + return n * { s: 1, m: 60, h: 3600, d: 86400 }[unit]!; +} + +// --------------------------------------------------------------------------- +// Deterministic RNG + distributions +// --------------------------------------------------------------------------- + +function mulberry32(seed: number): Rng { + let a = seed >>> 0; + return () => { + a |= 0; + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +function standardNormal(rng: Rng): number { + let u = 0; + let v = 0; + while (u === 0) u = rng(); + while (v === 0) v = rng(); + return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v); +} + +function lognormal(medianMs: number, sigma: number, rng: Rng): number { + return Math.exp(Math.log(Math.max(medianMs, 1)) + sigma * standardNormal(rng)); +} + +function poisson(lambda: number, rng: Rng): number { + if (lambda <= 0) return 0; + if (lambda > 30) return Math.max(0, Math.round(lambda + standardNormal(rng) * Math.sqrt(lambda))); + const L = Math.exp(-lambda); + let k = 0; + let p = 1; + do { + k++; + p *= rng(); + } while (p > L); + return k - 1; +} + +function formatChDateTime(date: Date): string { + return date.toISOString().slice(0, 19).replace("T", " "); +} + +// --------------------------------------------------------------------------- +// Scenarios +// --------------------------------------------------------------------------- + +const steady = (): QueueProfile[] => [ + { name: "emails", limit: () => 20, arrivals: (_b, r) => poisson(12, r), waitBaseMs: 40 }, + { name: "webhooks", limit: () => 15, arrivals: (_b, r) => poisson(9, r), waitBaseMs: 40 }, + { name: "reports", limit: () => 10, arrivals: (_b, r) => poisson(5, r), waitBaseMs: 60 }, +]; + +// periodic bursts every ~30 buckets +const bursty = (name: string, limit: number, base: number): QueueProfile => ({ + name, + limit: () => limit, + arrivals: (b, r) => poisson(b % 30 < 4 ? base * 5 : base, r), + waitBaseMs: 50, +}); + +const scenarios: Record<string, (totalBuckets: number, bucketSec: number) => Scenario> = { + steady: () => ({ + description: "all queues below capacity, no throttling", + envLimit: () => 60, + queues: steady(), + }), + + burst: () => ({ + description: "periodic arrival bursts -> backlog + wait spikes + throttling", + envLimit: () => 60, + queues: [bursty("ingest", 20, 6), bursty("transform", 20, 7)], + }), + + // Tela case: sum of per-queue limits far exceeds the env limit, so queues compete. + "over-allocated-env": () => ({ + description: "Sum(queue limits)=120 >> env limit=40; env saturates, queues env-limited", + envLimit: () => 40, + queues: Array.from({ length: 6 }, (_v, i) => ({ + name: `worker-${i + 1}`, + limit: () => 20, + arrivals: (_b: number, r: Rng) => poisson(14, r), + waitBaseMs: 50, + })), + }), + + "single-queue-starves-others": () => ({ + description: "one greedy queue consumes most of a small env limit, starving the rest", + envLimit: () => 30, + queues: [ + { name: "greedy", limit: () => 40, arrivals: (_b, r) => poisson(45, r), waitBaseMs: 60 }, + { name: "polite-1", limit: () => 10, arrivals: (_b, r) => poisson(6, r), waitBaseMs: 50 }, + { name: "polite-2", limit: () => 10, arrivals: (_b, r) => poisson(6, r), waitBaseMs: 50 }, + ], + }), + + "throttled-backlog": () => ({ + description: + "arrival rate persistently above the queue limit -> permanent backlog + throttling", + envLimit: () => 50, + queues: [ + { name: "overloaded", limit: () => 10, arrivals: (_b, r) => poisson(16, r), waitBaseMs: 80 }, + ], + }), + + "idle-sparse": () => ({ + description: "sparse arrivals with many empty buckets (carry-forward gaps)", + envLimit: () => 50, + queues: Array.from({ length: 4 }, (_v, i) => ({ + name: `sparse-${i + 1}`, + limit: () => 5, + arrivals: (_b: number, r: Rng) => (r() < 0.12 ? poisson(3, r) : 0), + waitBaseMs: 30, + sparse: true, + })), + }), + + "spike-then-drain": (totalBuckets) => ({ + description: "heavy arrivals for the first third, then zero; backlog builds then drains", + envLimit: () => 60, + queues: [ + { + name: "batch-job", + limit: () => 15, + arrivals: (b, r) => (b < totalBuckets / 3 ? poisson(30, r) : 0), + waitBaseMs: 70, + }, + ], + }), + + // Pagination + relevance-ranking design surface: one runaway queue, a busy-but-healthy + // head, a bursty middle, and a long sparse tail across 61 queues (the list pages at 25). + "many-queues": () => ({ + description: + "61 queues: one runaway, busy head, bursty middle, long sparse tail (pagination + ranking)", + envLimit: () => 150, + queues: [ + { name: "imports", limit: () => 8, arrivals: (_b, r) => poisson(14, r), waitBaseMs: 80 }, + ...["checkout", "notifications", "emails"].map((name, i) => ({ + name, + limit: () => 15, + arrivals: (_b: number, r: Rng) => poisson(7 + i, r), + waitBaseMs: 60, + })), + ...Array.from({ length: 12 }, (_v, i) => + bursty(`service-${String(i + 1).padStart(2, "0")}`, 10, 2) + ), + ...Array.from({ length: 20 }, (_v, i) => ({ + name: `job-${String(i + 1).padStart(2, "0")}`, + limit: () => 5, + arrivals: (_b: number, r: Rng) => poisson(1, r), + waitBaseMs: 40, + })), + ...Array.from({ length: 25 }, (_v, i) => ({ + name: `tenant-${String(i + 1).padStart(2, "0")}`, + limit: () => 3, + arrivals: (_b: number, r: Rng) => (r() < 0.05 ? poisson(2, r) : 0), + waitBaseMs: 30, + sparse: true, + })), + ], + }), + + // Per-tenant concurrency keys: a hog tenant periodically floods the queue and starves + // the others, so the CK charts (keys with backlog, most-starved wait) and the live + // per-key table on the queue detail page have something to show. Use with --usage. + "tenant-hotspot": () => ({ + description: + "CK queue where a hog tenant starves others: CK charts + live key table (use --usage)", + envLimit: () => 40, + queues: [ + { + name: "per-tenant", + limit: () => 10, + arrivals: (b, r) => poisson(b % 60 < 20 ? 25 : 8, r), + waitBaseMs: 60, + ck: { + backlogged: (b, r) => (b % 60 < 20 ? 6 + Math.round(r() * 6) : Math.round(r() * 3)), + maxWaitMs: (b, r) => + b % 60 < 20 + ? Math.round(lognormal(90_000, 0.5, r)) + : Math.round(lognormal(3_000, 0.6, r)), + }, + }, + { name: "background", limit: () => 10, arrivals: (_b, r) => poisson(5, r), waitBaseMs: 40 }, + ], + }), + + // Default: one env with a variety of queue behaviours + occasional env saturation. + mixed: (totalBuckets) => ({ + description: "variety of queue profiles in one env, with occasional env saturation", + envLimit: (b) => (b % 40 < 12 ? 45 : 70), // dips low periodically to flip env saturation + queues: [ + { name: "emails", limit: () => 20, arrivals: (_b, r) => poisson(12, r), waitBaseMs: 40 }, + bursty("webhooks", 20, 6), + { name: "reports", limit: () => 10, arrivals: (_b, r) => poisson(8, r), waitBaseMs: 80 }, + { + name: "cleanup", + limit: () => 5, + arrivals: (_b, r) => (r() < 0.12 ? poisson(3, r) : 0), + waitBaseMs: 30, + sparse: true, + }, + { + name: "nightly-batch", + limit: () => 15, + arrivals: (b, r) => (b < totalBuckets / 5 ? poisson(18, r) : 0), + waitBaseMs: 70, + }, + ], + }), +}; + +// --------------------------------------------------------------------------- +// Simulation +// --------------------------------------------------------------------------- + +type Ids = { organization_id: string; project_id: string; environment_id: string }; +const WAIT_SIGMA = 0.6; +const NACK_RATE = 0.02; +const DLQ_RATE = 0.004; + +type CounterOp = "enqueue" | "started" | "ack" | "nack" | "dlq"; +// Per-(queue, op) odometers, mirroring the production emitter: cumulative readings with a +// cum=0 baseline on the first one, so deltaSumTimestamp captures the 0->1 delta. +type CounterState = Record<CounterOp, number>[]; + +function counterRows( + counters: CounterState, + q: number, + ids: Ids, + queueName: string, + eventTime: string, + orderKey: () => number, + op: CounterOp, + wait_ms?: number +): QueueMetricsRawV1Input[] { + const rows: QueueMetricsRawV1Input[] = []; + if (counters[q][op] === 0) { + rows.push({ + ...ids, + queue_name: queueName, + event_time: eventTime, + op, + cumulative: 0, + order_key: orderKey(), + }); + } + counters[q][op] += 1; + rows.push({ + ...ids, + queue_name: queueName, + event_time: eventTime, + op, + cumulative: counters[q][op], + order_key: orderKey(), + ...(wait_ms !== undefined ? { wait_ms } : {}), + }); + return rows; +} + +function newCounterState(n: number): CounterState { + return Array.from({ length: n }, () => ({ enqueue: 0, started: 0, ack: 0, nack: 0, dlq: 0 })); +} + +// Per-key simulation for CK profiles: 12 tenants (tenant-01 is the hog, matching +// stageRedisUsage), per-tenant backlog drained round-robin, per-tenant odometers. +const CK_TENANT_COUNT = 12; +type CkSimState = { backlog: number[]; counters: Map<number, Record<CounterOp, number>> }; +const ckSim = new Map<number, CkSimState>(); + +function ckTenantName(t: number): string { + return `tenant-${String(t + 1).padStart(2, "0")}`; +} + +function ckCounterRows( + state: CkSimState, + tenant: number, + ids: Ids, + queueName: string, + eventTime: string, + orderKey: () => number, + op: CounterOp, + wait_ms?: number +): QueueMetricsRawV1Input[] { + let c = state.counters.get(tenant); + if (!c) { + c = { enqueue: 0, started: 0, ack: 0, nack: 0, dlq: 0 }; + state.counters.set(tenant, c); + } + const common = { + ...ids, + queue_name: queueName, + concurrency_key: ckTenantName(tenant), + event_time: eventTime, + }; + const rows: QueueMetricsRawV1Input[] = []; + if (c[op] === 0) rows.push({ ...common, op, cumulative: 0, order_key: orderKey() }); + c[op] += 1; + rows.push({ + ...common, + op, + cumulative: c[op], + order_key: orderKey(), + ...(wait_ms !== undefined ? { wait_ms } : {}), + }); + return rows; +} + +// Advance one bucket of the simulation for every queue, returning the raw rows to insert. +// `backlog` and `counters` are mutated in place so state carries across buckets (and into +// live mode). +function simulateBucket( + scenario: Scenario, + bucket: number, + bucketSec: number, + eventTime: string, + bucketEpochSec: number, + ids: Ids, + backlog: number[], + counters: CounterState, + rng: Rng +): QueueMetricsRawV1Input[] { + const envLimit = scenario.envLimit(bucket); + const n = scenario.queues.length; + + const limit = new Array(n); + const desired = new Array(n); + for (let q = 0; q < n; q++) { + limit[q] = scenario.queues[q].limit(bucket); + const arrivals = Math.min(500, scenario.queues[q].arrivals(bucket, rng)); + const prior = backlog[q]; // backlog carried from earlier buckets, before this bucket's arrivals + backlog[q] += arrivals; // arrivals join the backlog; recorded as enqueues below + (desired as any)[q] = { arrivals, prior, want: Math.min(limit[q], backlog[q]) }; + } + + // Env cap: if the queues collectively want more concurrency than the env allows, scale down. + const sumWant = desired.reduce((s: number, d: any) => s + d.want, 0); + const scale = sumWant > envLimit && sumWant > 0 ? envLimit / sumWant : 1; + + const running = new Array(n); + const queued = new Array(n); + let envRunning = 0; + let envQueued = 0; + for (let q = 0; q < n; q++) { + const d = desired[q] as any; + running[q] = Math.floor(d.want * scale); + queued[q] = backlog[q] - running[q]; + envRunning += running[q]; + envQueued += queued[q]; + } + + // Order keys are time-based (like the production stream ids) so appended runs and live + // mode stay monotonic; the per-bucket sequence keeps them unique within a bucket. + let bucketSeq = 0; + const orderKey = () => bucketEpochSec * 1_000_000 + bucketSeq++; + + const rows: QueueMetricsRawV1Input[] = []; + for (let q = 0; q < n; q++) { + const profile = scenario.queues[q]; + const started = running[q]; + const arrivals = (desired[q] as any).arrivals as number; + const prior = (desired[q] as any).prior as number; // depth a starting run actually queued behind + backlog[q] = queued[q]; // carry the unserved remainder forward + + if (profile.sparse && arrivals === 0 && started === 0 && prior === 0) { + continue; // fully idle: leave a gap so carry-forward is exercised + } + + // CK-health fields stay coherent with the depth: no queued runs means no backlogged keys. + const ckBacklogged = profile.ck + ? queued[q] > 0 + ? Math.max(1, Math.min(profile.ck.backlogged(bucket, rng), queued[q])) + : 0 + : undefined; + const ckMaxWaitMs = + profile.ck && ckBacklogged ? Math.round(profile.ck.maxWaitMs(bucket, rng)) : undefined; + + const gauge: QueueMetricsRawV1Input = { + ...ids, + queue_name: profile.name, + event_time: eventTime, + op: "gauge", + running: running[q], + queued: queued[q], + queue_limit: limit[q], + env_running: envRunning, + env_queued: envQueued, + env_limit: envLimit, + throttled: queued[q] > 0 && (running[q] >= limit[q] || scale < 1) ? 1 : 0, + ...(ckBacklogged !== undefined + ? { ck_backlogged: ckBacklogged, ck_max_wait_ms: ckMaxWaitMs ?? 0 } + : {}), + }; + rows.push(gauge); + + for (let a = 0; a < arrivals; a++) { + rows.push(...counterRows(counters, q, ids, profile.name, eventTime, orderKey, "enqueue")); + } + + // Per-key rows for CK profiles: assign arrivals hog-weighted, drain round-robin + // (fair share), then emit per-tenant odometers + a per-key gauge per active tenant. + if (profile.ck) { + let ckq = ckSim.get(q); + if (!ckq) { + ckq = { backlog: new Array(CK_TENANT_COUNT).fill(0), counters: new Map() }; + ckSim.set(q, ckq); + } + const hogShare = bucket % 60 < 20 ? 0.6 : 0.15; + const arrivalsPerTenant = new Array(CK_TENANT_COUNT).fill(0); + for (let a = 0; a < arrivals; a++) { + const t = rng() < hogShare ? 0 : 1 + Math.floor(rng() * (CK_TENANT_COUNT - 1)); + arrivalsPerTenant[t]++; + ckq.backlog[t]++; + } + const drainedPerTenant = new Array(CK_TENANT_COUNT).fill(0); + let remaining = started; + while (remaining > 0 && ckq.backlog.some((v) => v > 0)) { + for (let t = 0; t < CK_TENANT_COUNT && remaining > 0; t++) { + if (ckq.backlog[t] > 0) { + ckq.backlog[t]--; + drainedPerTenant[t]++; + remaining--; + } + } + } + for (let t = 0; t < CK_TENANT_COUNT; t++) { + const fairShare = Math.max(1, limit[q] / CK_TENANT_COUNT); + const ckMedianWait = profile.waitBaseMs + (ckq.backlog[t] / fairShare) * bucketSec * 1000; + for (let a = 0; a < arrivalsPerTenant[t]; a++) { + rows.push(...ckCounterRows(ckq, t, ids, profile.name, eventTime, orderKey, "enqueue")); + } + for (let d = 0; d < drainedPerTenant[t]; d++) { + rows.push( + ...ckCounterRows( + ckq, + t, + ids, + profile.name, + eventTime, + orderKey, + "started", + Math.round(lognormal(ckMedianWait, WAIT_SIGMA, rng)) + ) + ); + rows.push(...ckCounterRows(ckq, t, ids, profile.name, eventTime, orderKey, "ack")); + } + if (ckq.backlog[t] > 0 || drainedPerTenant[t] > 0) { + rows.push({ + ...ids, + queue_name: profile.name, + concurrency_key: ckTenantName(t), + event_time: eventTime, + op: "gauge", + queued: ckq.backlog[t], + running: drainedPerTenant[t], + }); + } + } + } + + const medianWait = profile.waitBaseMs + (prior / Math.max(limit[q], 1)) * bucketSec * 1000; + for (let s = 0; s < started; s++) { + rows.push( + ...counterRows( + counters, + q, + ids, + profile.name, + eventTime, + orderKey, + "started", + Math.round(lognormal(medianWait, WAIT_SIGMA, rng)) + ) + ); + const roll = rng(); + const op: CounterOp = roll < DLQ_RATE ? "dlq" : roll < DLQ_RATE + NACK_RATE ? "nack" : "ack"; + rows.push(...counterRows(counters, q, ids, profile.name, eventTime, orderKey, op)); + } + } + return rows; +} + +// --------------------------------------------------------------------------- +// ClickHouse +// --------------------------------------------------------------------------- + +function clickhouse(): ClickHouse { + const clickhouseUrl = process.env.CLICKHOUSE_URL ?? process.env.EVENTS_CLICKHOUSE_URL; + if (!clickhouseUrl) { + console.error("CLICKHOUSE_URL not set"); + process.exit(1); + } + const url = new URL(clickhouseUrl); + // Allowlist local hosts only (this script TRUNCATEs), and never echo the URL (it carries creds). + const localHosts = new Set(["localhost", "127.0.0.1", "::1", "0.0.0.0"]); + if (!localHosts.has(url.hostname)) { + console.error(`Refusing to run against a non-local ClickHouse host: ${url.hostname}`); + process.exit(1); + } + url.searchParams.delete("secure"); + return new ClickHouse({ url: url.toString(), name: "queue-metrics-simulator" }); +} + +async function insertBatched(ch: ClickHouse, rows: QueueMetricsRawV1Input[], nonce: string) { + const BATCH = 25_000; + for (let i = 0; i < rows.length; i += BATCH) { + const slice = rows.slice(i, i + BATCH); + const [error] = await ch.queueMetrics.insertRaw(slice, { + params: { clickhouse_settings: { insert_deduplication_token: `${nonce}:${i}` } }, + }); + if (error) { + console.error("insert failed:", error.message); + process.exit(1); + } + } +} + +async function resetEnv(ch: ClickHouse, environmentId: string) { + const raw = ( + ch.writer as unknown as { client: { command: (a: { query: string }) => Promise<unknown> } } + ).client; + for (const table of [ + "queue_metrics_raw_v1", + "queue_metrics_v1", + "queue_metrics_5m_v1", + "env_metrics_v1", + "queue_metrics_ck_v1", + ]) { + await raw.command({ + query: `DELETE FROM trigger_dev.${table} WHERE environment_id = '${environmentId}'`, + }); + } + console.log(`Reset queue metrics for environment ${environmentId}`); +} + +// Fake running counts in the run-queue Redis (Running column + allocation usage bars). +// Reconciled every run: staged with --usage, cleared otherwise. +async function stageRedisUsage(scenario: Scenario, ids: Ids, seed: number, clear: boolean) { + const host = process.env.RUN_ENGINE_RUN_QUEUE_REDIS_HOST ?? process.env.REDIS_HOST ?? "localhost"; + const port = Number( + process.env.RUN_ENGINE_RUN_QUEUE_REDIS_PORT ?? process.env.REDIS_PORT ?? 6379 + ); + const localHosts = new Set(["localhost", "127.0.0.1", "::1", "0.0.0.0"]); + if (!localHosts.has(host)) { + console.warn(`Skipping Redis usage staging on a non-local host: ${host}`); + return; + } + try { + const { createRedisClient } = await import("@internal/redis"); + const redis = createRedisClient({ host, port }); + const rng = mulberry32(seed + 1); + const prefix = "engine:runqueue:"; + const logicalBase = `{org:${ids.organization_id}}:proj:${ids.project_id}:env:${ids.environment_id}:queue:`; + const base = `${prefix}${logicalBase}`; + + // Env-level structures the Queues list "Queued"/"Running" blocks read: + // lengthOfEnvQueue -> ZCARD(envQueueKey) (ZSET, no proj section) + // concurrencyOfEnvQueue -> SCARD(envCurrentDequeuedKey) (SET) + // We accumulate the per-queue staged counts below and stage these so the blocks + // equal the table's per-queue sums instead of showing 0/0. + const envQueueKey = `${prefix}{org:${ids.organization_id}}:env:${ids.environment_id}`; + const envCurrentDequeuedKey = `${prefix}{org:${ids.organization_id}}:proj:${ids.project_id}:env:${ids.environment_id}:currentDequeued`; + await redis.del(envQueueKey, envCurrentDequeuedKey); + // Table Queued = ZCARD(base) + lengthCounter (base zset unstaged -> only CK queues + // contribute their lengthCounter). Table Running = SCARD(currentDequeued) per queue. + let envQueuedTotal = 0; + let envRunningTotal = 0; + + for (const [q, profile] of scenario.queues.entries()) { + const key = `${base}${profile.name}:currentDequeued`; + await redis.del(key); + + // CK staging (ckIndex + per-key subqueues) feeds the live per-key table on the queue + // detail page. Members are stored unprefixed, exactly like the run-queue Lua does. + const ckIndexKey = `${base}${profile.name}:ckIndex`; + const lengthCounterKey = `${base}${profile.name}:lengthCounter`; + const staleMembers = await redis.zrange(ckIndexKey, 0, -1); + for (const member of staleMembers) { + await redis.del(`${prefix}${member}`, `${prefix}${member}:currentConcurrency`); + } + await redis.del(ckIndexKey, lengthCounterKey); + + if (clear) continue; + const limit = profile.limit(0); + // First queue rides at/over its limit, the rest at 30-90%, sparse mostly idle. + const count = profile.sparse + ? rng() < 0.3 + ? 1 + : 0 + : q === 0 + ? limit + Math.round(rng() * 2) + : Math.round(limit * (0.3 + 0.6 * rng())); + if (count > 0) { + await redis.sadd(key, ...Array.from({ length: count }, (_v, i) => `sim_run_${i}`)); + } + envRunningTotal += count; + + if (profile.ck) { + const now = Date.now(); + const tenants = 12; + let totalCkQueued = 0; + for (let t = 1; t <= tenants; t++) { + const tenant = `tenant-${String(t).padStart(2, "0")}`; + const member = `${logicalBase}${profile.name}:ck:${tenant}`; + const hog = t === 1; + const queuedCount = hog ? 40 : 1 + Math.round(rng() * 5); + const runningCount = hog ? limit : Math.round(rng() * 2); + const oldestAgeMs = hog ? 15 * 60_000 : 5_000 + Math.round(rng() * 55_000); + const zargs: Array<string | number> = []; + for (let i = 0; i < queuedCount; i++) { + zargs.push(now - oldestAgeMs + i * 250, `sim_${tenant}_run_${i}`); + } + await redis.zadd(`${prefix}${member}`, ...zargs); + if (runningCount > 0) { + await redis.sadd( + `${prefix}${member}:currentConcurrency`, + ...Array.from({ length: runningCount }, (_v, i) => `sim_${tenant}_running_${i}`) + ); + } + await redis.zadd(ckIndexKey, now - oldestAgeMs, member); + totalCkQueued += queuedCount; + } + // The aggregate "Queued now" reads ZCARD(base) + this counter; keep them coherent. + await redis.set(lengthCounterKey, totalCkQueued, "EX", 24 * 3600); + envQueuedTotal += totalCkQueued; + } + } + + // Stage the env-level structures so the list-page blocks match the table sums. + // Members are unique across queues (each per-queue set uses its own key, but the + // env set/zset needs distinct members to reach the summed cardinality). + if (!clear) { + if (envRunningTotal > 0) { + await redis.sadd( + envCurrentDequeuedKey, + ...Array.from({ length: envRunningTotal }, (_v, i) => `sim_env_run_${i}`) + ); + } + if (envQueuedTotal > 0) { + const now = Date.now(); + const zargs: Array<string | number> = []; + for (let i = 0; i < envQueuedTotal; i++) zargs.push(now + i, `sim_env_queued_${i}`); + await redis.zadd(envQueueKey, ...zargs); + } + } + + await redis.quit(); + console.log( + clear + ? "Cleared staged Redis usage." + : "Staged fake running counts in Redis (Running column + allocation usage bars)." + ); + } catch (error) { + console.warn("Redis usage staging skipped:", error instanceof Error ? error.message : error); + } +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +// Make the synthetic project a V2 engine project with a current dev worker + a Postgres +// TaskQueue per simulated queue, so the /queues list renders the V2 table (it pages from +// Postgres and gates on engine version; ClickHouse only holds the metrics). +async function ensureTaskQueues( + scenario: Scenario, + projectId: string, + runtimeEnvironmentId: string +) { + await prisma.project.update({ where: { id: projectId }, data: { engine: "V2" } }); + + await prisma.backgroundWorker.upsert({ + where: { + projectId_runtimeEnvironmentId_version: { + projectId, + runtimeEnvironmentId, + version: "queue-metrics-sim", + }, + }, + update: {}, + create: { + friendlyId: generateFriendlyId("worker"), + engine: "V2", + contentHash: "queue-metrics-sim", + sdkVersion: "4.0.0", + cliVersion: "4.0.0", + projectId, + runtimeEnvironmentId, + version: "queue-metrics-sim", + metadata: {}, + }, + }); + + for (const profile of scenario.queues) { + const concurrencyLimit = profile.limit(0); + await prisma.taskQueue.upsert({ + where: { runtimeEnvironmentId_name: { runtimeEnvironmentId, name: profile.name } }, + create: { + friendlyId: generateFriendlyId("queue"), + version: "V2", + name: profile.name, + orderableName: profile.name, + concurrencyLimit, + runtimeEnvironmentId, + projectId, + type: "NAMED", + }, + // Reset any dashboard override left from manual testing: re-seeding overwrites the + // materialized concurrencyLimit, so a surviving override percent/base would contradict it + // (e.g. "10 (77%)" with an env limit of 25). + update: { + concurrencyLimit, + concurrencyLimitBase: null, + concurrencyLimitOverridePercent: null, + concurrencyLimitOverriddenAt: null, + concurrencyLimitOverriddenBy: null, + }, + }); + } + + // Drop queues left over from a previously seeded scenario so switching scenarios + // does not leave metric-less rows in the list. + const { count: pruned } = await prisma.taskQueue.deleteMany({ + where: { + runtimeEnvironmentId, + name: { notIn: scenario.queues.map((q) => q.name) }, + }, + }); + console.log( + `Ensured ${scenario.queues.length} task queues in Postgres${pruned > 0 ? `, pruned ${pruned} stale` : ""}.` + ); +} + +function printHelp() { + const lines = Object.entries(scenarios).map( + ([name, build]) => ` ${name.padEnd(28)}${build(720, 10).description}` + ); + console.log(`Queue metrics simulator: seeds a synthetic tenant with realistic queue metrics. + +Usage: pnpm --filter webapp run db:seed:queue-metrics -- [flags] + +Flags: + --scenario <name> which scenario to seed (default: mixed) + --project <name> project to seed into (default: ${PROJECT_NAME}); use one + project per scenario to browse them side by side + --window <dur> how much history to backfill, e.g. 30m, 6h, 1d (default: 2h) + --bucket <sec> seconds per simulated bucket (default: 10) + --seed <n> RNG seed for reproducible data (default: 1) + --usage stage fake running counts in Redis so the Running column and + the Allocation tab's usage bars have data (cleared when omitted) + --live after backfilling, keep appending one bucket per interval + --reset clear this environment's metrics before seeding + --reset-only clear and exit without seeding + --help this text + +Scenarios: +${lines.join("\n")} + +Example designer setup (one project per scenario): + pnpm --filter webapp run db:seed:queue-metrics -- --scenario mixed --reset + pnpm --filter webapp run db:seed:queue-metrics -- --scenario many-queues --project qm-many-queues --reset + pnpm --filter webapp run db:seed:queue-metrics -- --scenario throttled-backlog --project qm-throttled --reset + pnpm --filter webapp run db:seed:queue-metrics -- --scenario tenant-hotspot --project qm-tenants --usage --reset`); +} + +async function main() { + const flags = parseArgs(process.argv.slice(2)); + if (flags.help === "true") { + printHelp(); + process.exit(0); + } + const scenarioName = flags.scenario ?? "mixed"; + const build = scenarios[scenarioName]; + if (!build) { + console.error( + `Unknown scenario "${scenarioName}". Options: ${Object.keys(scenarios).join(", ")}` + ); + process.exit(1); + } + const bucketSec = Number(flags.bucket ?? 10); + if (!Number.isFinite(bucketSec) || bucketSec <= 0) { + console.error(`--bucket must be a positive number of seconds, got: ${flags.bucket}`); + process.exit(1); + } + const windowSec = parseDuration(flags.window ?? "2h"); + const totalBuckets = Math.floor(windowSec / bucketSec); + if (!Number.isFinite(totalBuckets) || totalBuckets <= 0) { + console.error( + `--window must be longer than --bucket (got ${windowSec}s window, ${bucketSec}s bucket)` + ); + process.exit(1); + } + const seed = Number(flags.seed ?? 1); + const live = flags.live === "true"; + + const user = await prisma.user.findUnique({ where: { email: "local@trigger.dev" } }); + if (!user) { + console.error("User local@trigger.dev not found. Run `pnpm run db:seed` first."); + process.exit(1); + } + + let org = await prisma.organization.findFirst({ + where: { title: ORG_TITLE, members: { some: { userId: user.id } } }, + }); + if (!org) + org = await createOrganization({ title: ORG_TITLE, userId: user.id, companySize: "1-10" }); + + const projectName = flags.project ?? PROJECT_NAME; + let project = await prisma.project.findFirst({ + where: { name: projectName, organizationId: org.id }, + }); + if (!project) { + project = await createProject({ + organizationSlug: org.slug, + name: projectName, + userId: user.id, + version: "v3", + }); + } + + const runtimeEnv = await prisma.runtimeEnvironment.findFirst({ + where: { projectId: project.id, type: "DEVELOPMENT" }, + }); + if (!runtimeEnv) { + console.error("No DEVELOPMENT environment found for project."); + process.exit(1); + } + + const ids: Ids = { + organization_id: org.id, + project_id: project.id, + environment_id: runtimeEnv.id, + }; + const ch = clickhouse(); + const nonce = `qmsim-${Date.now()}-${seed}`; + + if (flags.reset === "true" || flags["reset-only"] === "true") { + await resetEnv(ch, runtimeEnv.id); + if (flags["reset-only"] === "true") { + await ch.close(); + process.exit(0); + } + } + + const scenario = build(totalBuckets, bucketSec); + await ensureTaskQueues(scenario, project.id, runtimeEnv.id); + await stageRedisUsage(scenario, ids, seed, flags.usage !== "true"); + const rng = mulberry32(seed); + const backlog = new Array(scenario.queues.length).fill(0); + + console.log(`Scenario "${scenarioName}": ${scenario.description}`); + console.log( + `Backfilling ${totalBuckets} x ${bucketSec}s buckets (${flags.window ?? "2h"}) for ${scenario.queues.length} queues...` + ); + + // Backfill: buckets from (now - window) up to now, aligned to the bucket grid. + const nowBucket = Math.floor(Date.now() / 1000 / bucketSec) * bucketSec; + const startBucket = nowBucket - totalBuckets * bucketSec; + const counters = newCounterState(scenario.queues.length); + const rows: QueueMetricsRawV1Input[] = []; + for (let b = 0; b < totalBuckets; b++) { + const bucketEpochSec = startBucket + b * bucketSec; + const eventTime = formatChDateTime(new Date(bucketEpochSec * 1000)); + rows.push( + ...simulateBucket( + scenario, + b, + bucketSec, + eventTime, + bucketEpochSec, + ids, + backlog, + counters, + rng + ) + ); + } + await insertBatched(ch, rows, nonce); + console.log(`Inserted ${rows.length} raw rows.`); + + // Merge the AggregatingMergeTree partials so argMax "current value" widgets read cleanly. + // The real pipeline relies on background merges; the simulator forces it for a tidy demo. + const raw = ( + ch.writer as unknown as { client: { command: (a: { query: string }) => Promise<unknown> } } + ).client; + await raw.command({ query: `OPTIMIZE TABLE trigger_dev.queue_metrics_v1 FINAL` }); + await raw.command({ query: `OPTIMIZE TABLE trigger_dev.queue_metrics_5m_v1 FINAL` }); + await raw.command({ query: `OPTIMIZE TABLE trigger_dev.env_metrics_v1 FINAL` }); + await raw.command({ query: `OPTIMIZE TABLE trigger_dev.queue_metrics_ck_v1 FINAL` }); + + const origin = process.env.APP_ORIGIN ?? "http://localhost:3030"; + console.log( + `\nQueues dashboard: ${origin}/orgs/${org.slug}/projects/${project.slug}/env/dev/dashboards/queues` + ); + + if (live) { + console.log(`\nLive mode: appending one bucket every ${bucketSec}s (Ctrl-C to stop)...`); + let b = totalBuckets; + // eslint-disable-next-line no-constant-condition + while (true) { + await new Promise((r) => setTimeout(r, bucketSec * 1000)); + const bucketEpochSec = Math.floor(Date.now() / 1000 / bucketSec) * bucketSec; + const eventTime = formatChDateTime(new Date(bucketEpochSec * 1000)); + const liveRows = simulateBucket( + scenario, + b, + bucketSec, + eventTime, + bucketEpochSec, + ids, + backlog, + counters, + rng + ); + await insertBatched(ch, liveRows, `${nonce}:live:${b}`); + console.log(`bucket ${b}: ${liveRows.length} rows @ ${eventTime}`); + b++; + } + } + + await ch.close(); + process.exit(0); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/apps/webapp/test/themePreference.test.ts b/apps/webapp/test/themePreference.test.ts new file mode 100644 index 00000000000..420cb2878a4 --- /dev/null +++ b/apps/webapp/test/themePreference.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; +import { parseDashboardPreferences } from "~/utils/dashboardPreferences"; +import { normalizeThemePreference, type ThemePreference } from "~/utils/themePreference"; + +const VALID_THEMES: ThemePreference[] = ["classic", "system", "dark", "light"]; + +describe("normalizeThemePreference", () => { + it("returns each valid value unchanged", () => { + for (const theme of VALID_THEMES) { + expect(normalizeThemePreference(theme)).toBe(theme); + } + }); + + it("falls back to dark for legacy/unknown values", () => { + expect(normalizeThemePreference("solarized")).toBe("dark"); + expect(normalizeThemePreference("")).toBe("dark"); + expect(normalizeThemePreference(42)).toBe("dark"); + expect(normalizeThemePreference(null)).toBe("dark"); + }); + + it("falls back to dark for undefined", () => { + expect(normalizeThemePreference(undefined)).toBe("dark"); + }); +}); + +describe("DashboardPreferences theme schema", () => { + it("accepts all four theme values", () => { + for (const theme of VALID_THEMES) { + const result = parseDashboardPreferences({ version: "1", projects: {}, theme }); + expect(result.theme).toBe(theme); + } + }); + + it("accepts preferences without a theme", () => { + const result = parseDashboardPreferences({ version: "1", projects: {} }); + expect(result.theme).toBeUndefined(); + }); + + it("drops an invalid theme without erasing the rest of the preferences", () => { + const result = parseDashboardPreferences({ + version: "1", + projects: {}, + theme: "neon", + contrast: 999, + currentProjectId: "proj_123", + sideMenu: { isCollapsed: true }, + }); + expect(result.theme).toBeUndefined(); + expect(result.contrast).toBeUndefined(); + expect(result.currentProjectId).toBe("proj_123"); + expect(result.sideMenu?.isCollapsed).toBe(true); + }); +}); diff --git a/internal-packages/llm-model-catalog/src/defaultPrices.ts b/internal-packages/llm-model-catalog/src/defaultPrices.ts index 982b6b2ec15..fb347c2bef6 100644 --- a/internal-packages/llm-model-catalog/src/defaultPrices.ts +++ b/internal-packages/llm-model-catalog/src/defaultPrices.ts @@ -6,3106 +6,3091 @@ import type { DefaultModelDefinition } from "./types.js"; export const defaultModelPrices: DefaultModelDefinition[] = [ { - modelName: "gpt-4o", - matchPattern: "(?i)^(openai/)?(gpt-4o)$", - startDate: "2024-05-13T23:15:07.670Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.0000025, - input_cached_tokens: 0.00000125, - input_cache_read: 0.00000125, - output: 0.00001, - }, - }, - ], - }, - { - modelName: "gpt-4o-2024-05-13", - matchPattern: "(?i)^(openai/)?(gpt-4o-2024-05-13)$", - startDate: "2024-05-13T23:15:07.670Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000005, - output: 0.000015, - }, - }, - ], - }, - { - modelName: "gpt-4-1106-preview", - matchPattern: "(?i)^(openai/)?(gpt-4-1106-preview)$", - startDate: "2024-04-23T10:37:17.092Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00001, - output: 0.00003, - }, - }, - ], - }, - { - modelName: "gpt-4-turbo-vision", - matchPattern: "(?i)^(openai/)?(gpt-4(-\\d{4})?-vision-preview)$", - startDate: "2024-01-24T10:19:21.693Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00001, - output: 0.00003, - }, - }, - ], - }, - { - modelName: "gpt-4-32k", - matchPattern: "(?i)^(openai/)?(gpt-4-32k)$", - startDate: "2024-01-24T10:19:21.693Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00006, - output: 0.00012, - }, - }, - ], - }, - { - modelName: "gpt-4-32k-0613", - matchPattern: "(?i)^(openai/)?(gpt-4-32k-0613)$", - startDate: "2024-01-24T10:19:21.693Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00006, - output: 0.00012, - }, - }, - ], - }, - { - modelName: "gpt-3.5-turbo-1106", - matchPattern: "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-1106)$", - startDate: "2024-01-24T10:19:21.693Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000001, - output: 0.000002, - }, - }, - ], - }, - { - modelName: "gpt-3.5-turbo-0613", - matchPattern: "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-0613)$", - startDate: "2024-01-24T10:19:21.693Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.0000015, - output: 0.000002, - }, - }, - ], - }, - { - modelName: "gpt-4-0613", - matchPattern: "(?i)^(openai/)?(gpt-4-0613)$", - startDate: "2024-01-24T10:19:21.693Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00003, - output: 0.00006, - }, - }, - ], - }, - { - modelName: "gpt-3.5-turbo-instruct", - matchPattern: "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-instruct)$", - startDate: "2024-01-24T10:19:21.693Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.0000015, - output: 0.000002, - }, - }, - ], - }, - { - modelName: "text-ada-001", - matchPattern: "(?i)^(text-ada-001)$", - startDate: "2024-01-24T18:18:50.861Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - total: 0.000004, - }, - }, - ], - }, - { - modelName: "text-babbage-001", - matchPattern: "(?i)^(text-babbage-001)$", - startDate: "2024-01-24T18:18:50.861Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - total: 5e-7, - }, - }, - ], - }, - { - modelName: "text-curie-001", - matchPattern: "(?i)^(text-curie-001)$", - startDate: "2024-01-24T18:18:50.861Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - total: 0.00002, - }, - }, - ], - }, - { - modelName: "text-davinci-001", - matchPattern: "(?i)^(text-davinci-001)$", - startDate: "2024-01-24T18:18:50.861Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - total: 0.00002, - }, - }, - ], - }, - { - modelName: "text-davinci-002", - matchPattern: "(?i)^(text-davinci-002)$", - startDate: "2024-01-24T18:18:50.861Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - total: 0.00002, - }, - }, - ], - }, - { - modelName: "text-davinci-003", - matchPattern: "(?i)^(text-davinci-003)$", - startDate: "2024-01-24T18:18:50.861Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - total: 0.00002, - }, - }, - ], - }, - { - modelName: "text-embedding-ada-002-v2", - matchPattern: "(?i)^(text-embedding-ada-002-v2)$", - startDate: "2024-01-24T18:18:50.861Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - total: 1e-7, - }, - }, - ], - }, - { - modelName: "text-embedding-ada-002", - matchPattern: "(?i)^(text-embedding-ada-002)$", - startDate: "2024-01-24T18:18:50.861Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - total: 1e-7, - }, - }, - ], - }, - { - modelName: "gpt-3.5-turbo-16k-0613", - matchPattern: "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-16k-0613)$", - startDate: "2024-02-03T17:29:57.350Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000003, - output: 0.000004, - }, - }, - ], - }, - { - modelName: "gpt-3.5-turbo-0301", - matchPattern: "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-0301)$", - startDate: "2024-01-24T10:19:21.693Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000002, - output: 0.000002, - }, - }, - ], - }, - { - modelName: "gpt-4-32k-0314", - matchPattern: "(?i)^(openai/)?(gpt-4-32k-0314)$", - startDate: "2024-01-24T10:19:21.693Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00006, - output: 0.00012, - }, - }, - ], - }, - { - modelName: "gpt-4-0314", - matchPattern: "(?i)^(openai/)?(gpt-4-0314)$", - startDate: "2024-01-24T10:19:21.693Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00003, - output: 0.00006, - }, - }, - ], - }, - { - modelName: "gpt-4", - matchPattern: "(?i)^(openai/)?(gpt-4)$", - startDate: "2024-01-24T10:19:21.693Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00003, - output: 0.00006, - }, - }, - ], - }, - { - modelName: "claude-instant-1.2", - matchPattern: "(?i)^(anthropic/)?(claude-instant-1.2)$", - startDate: "2024-01-30T15:44:13.447Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00000163, - output: 0.00000551, - }, - }, - ], - }, - { - modelName: "claude-2.0", - matchPattern: "(?i)^(anthropic/)?(claude-2.0)$", - startDate: "2024-01-30T15:44:13.447Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000008, - output: 0.000024, - }, - }, - ], - }, - { - modelName: "claude-2.1", - matchPattern: "(?i)^(anthropic/)?(claude-2.1)$", - startDate: "2024-01-30T15:44:13.447Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000008, - output: 0.000024, - }, - }, - ], - }, - { - modelName: "claude-1.3", - matchPattern: "(?i)^(anthropic/)?(claude-1.3)$", - startDate: "2024-01-30T15:44:13.447Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000008, - output: 0.000024, - }, - }, - ], - }, - { - modelName: "claude-1.2", - matchPattern: "(?i)^(anthropic/)?(claude-1.2)$", - startDate: "2024-01-30T15:44:13.447Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000008, - output: 0.000024, - }, - }, - ], - }, - { - modelName: "claude-1.1", - matchPattern: "(?i)^(anthropic/)?(claude-1.1)$", - startDate: "2024-01-30T15:44:13.447Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000008, - output: 0.000024, - }, - }, - ], - }, - { - modelName: "claude-instant-1", - matchPattern: "(?i)^(anthropic/)?(claude-instant-1)$", - startDate: "2024-01-30T15:44:13.447Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00000163, - output: 0.00000551, - }, - }, - ], - }, - { - modelName: "babbage-002", - matchPattern: "(?i)^(babbage-002)$", - startDate: "2024-01-26T17:35:21.129Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 4e-7, - output: 0.0000016, - }, - }, - ], - }, - { - modelName: "davinci-002", - matchPattern: "(?i)^(davinci-002)$", - startDate: "2024-01-26T17:35:21.129Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000006, - output: 0.000012, - }, - }, - ], - }, - { - modelName: "text-embedding-3-small", - matchPattern: "(?i)^(text-embedding-3-small)$", - startDate: "2024-01-26T17:35:21.129Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - total: 2e-8, - }, - }, - ], - }, - { - modelName: "text-embedding-3-large", - matchPattern: "(?i)^(text-embedding-3-large)$", - startDate: "2024-01-26T17:35:21.129Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - total: 1.3e-7, - }, - }, - ], - }, - { - modelName: "gpt-3.5-turbo-0125", - matchPattern: "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-0125)$", - startDate: "2024-01-26T17:35:21.129Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 5e-7, - output: 0.0000015, - }, - }, - ], - }, - { - modelName: "gpt-3.5-turbo", - matchPattern: "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo)$", - startDate: "2024-02-13T12:00:37.424Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 5e-7, - output: 0.0000015, - }, - }, - ], - }, - { - modelName: "gpt-4-0125-preview", - matchPattern: "(?i)^(openai/)?(gpt-4-0125-preview)$", - startDate: "2024-01-26T17:35:21.129Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00001, - output: 0.00003, - }, - }, - ], - }, - { - modelName: "ft:gpt-3.5-turbo-1106", - matchPattern: "(?i)^(ft:)(gpt-3.5-turbo-1106:)(.+)(:)(.*)(:)(.+)$", - startDate: "2024-01-31T13:25:02.141Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000003, - output: 0.000006, - }, - }, - ], - }, - { - modelName: "ft:gpt-3.5-turbo-0613", - matchPattern: "(?i)^(ft:)(gpt-3.5-turbo-0613:)(.+)(:)(.*)(:)(.+)$", - startDate: "2024-01-31T13:25:02.141Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000012, - output: 0.000016, - }, - }, - ], - }, - { - modelName: "ft:davinci-002", - matchPattern: "(?i)^(ft:)(davinci-002:)(.+)(:)(.*)(:)(.+)$$", - startDate: "2024-01-31T13:25:02.141Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000012, - output: 0.000012, - }, - }, - ], - }, - { - modelName: "ft:babbage-002", - matchPattern: "(?i)^(ft:)(babbage-002:)(.+)(:)(.*)(:)(.+)$$", - startDate: "2024-01-31T13:25:02.141Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.0000016, - output: 0.0000016, - }, - }, - ], - }, - { - modelName: "chat-bison", - matchPattern: "(?i)^(chat-bison)(@[a-zA-Z0-9]+)?$", - startDate: "2024-01-31T13:25:02.141Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 2.5e-7, - output: 5e-7, - }, - }, - ], - }, - { - modelName: "codechat-bison-32k", - matchPattern: "(?i)^(codechat-bison-32k)(@[a-zA-Z0-9]+)?$", - startDate: "2024-01-31T13:25:02.141Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 2.5e-7, - output: 5e-7, - }, - }, - ], - }, - { - modelName: "codechat-bison", - matchPattern: "(?i)^(codechat-bison)(@[a-zA-Z0-9]+)?$", - startDate: "2024-01-31T13:25:02.141Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 2.5e-7, - output: 5e-7, - }, - }, - ], - }, - { - modelName: "text-bison-32k", - matchPattern: "(?i)^(text-bison-32k)(@[a-zA-Z0-9]+)?$", - startDate: "2024-01-31T13:25:02.141Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 2.5e-7, - output: 5e-7, - }, - }, - ], - }, - { - modelName: "chat-bison-32k", - matchPattern: "(?i)^(chat-bison-32k)(@[a-zA-Z0-9]+)?$", - startDate: "2024-01-31T13:25:02.141Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 2.5e-7, - output: 5e-7, - }, - }, - ], - }, - { - modelName: "text-unicorn", - matchPattern: "(?i)^(text-unicorn)(@[a-zA-Z0-9]+)?$", - startDate: "2024-01-31T13:25:02.141Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.0000025, - output: 0.0000075, - }, - }, - ], - }, - { - modelName: "text-bison", - matchPattern: "(?i)^(text-bison)(@[a-zA-Z0-9]+)?$", - startDate: "2024-01-31T13:25:02.141Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 2.5e-7, - output: 5e-7, - }, - }, - ], - }, - { - modelName: "textembedding-gecko", - matchPattern: "(?i)^(textembedding-gecko)(@[a-zA-Z0-9]+)?$", - startDate: "2024-01-31T13:25:02.141Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - total: 1e-7, - }, - }, - ], - }, - { - modelName: "textembedding-gecko-multilingual", - matchPattern: "(?i)^(textembedding-gecko-multilingual)(@[a-zA-Z0-9]+)?$", - startDate: "2024-01-31T13:25:02.141Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - total: 1e-7, - }, - }, - ], - }, - { - modelName: "code-gecko", - matchPattern: "(?i)^(code-gecko)(@[a-zA-Z0-9]+)?$", - startDate: "2024-01-31T13:25:02.141Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 2.5e-7, - output: 5e-7, - }, - }, - ], - }, - { - modelName: "code-bison", - matchPattern: "(?i)^(code-bison)(@[a-zA-Z0-9]+)?$", - startDate: "2024-01-31T13:25:02.141Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 2.5e-7, - output: 5e-7, - }, - }, - ], - }, - { - modelName: "code-bison-32k", - matchPattern: "(?i)^(code-bison-32k)(@[a-zA-Z0-9]+)?$", - startDate: "2024-01-31T13:25:02.141Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 2.5e-7, - output: 5e-7, - }, - }, - ], - }, - { - modelName: "gpt-3.5-turbo-16k", - matchPattern: "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-16k)$", - startDate: "2024-02-13T12:00:37.424Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 5e-7, - output: 0.0000015, - }, - }, - ], - }, - { - modelName: "gpt-4-turbo-preview", - matchPattern: "(?i)^(openai/)?(gpt-4-turbo-preview)$", - startDate: "2024-02-15T21:21:50.947Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00001, - output: 0.00003, - }, - }, - ], - }, - { - modelName: "claude-3-opus-20240229", - matchPattern: - "(?i)^(anthropic/)?(claude-3-opus-20240229|anthropic\\.claude-3-opus-20240229-v1:0|claude-3-opus@20240229)$", - startDate: "2024-03-07T17:55:38.139Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000015, - output: 0.000075, - }, - }, - ], - }, - { - modelName: "claude-3-sonnet-20240229", - matchPattern: - "(?i)^(anthropic/)?(claude-3-sonnet-20240229|anthropic\\.claude-3-sonnet-20240229-v1:0|claude-3-sonnet@20240229)$", - startDate: "2024-03-07T17:55:38.139Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000003, - input_tokens: 0.000003, - output: 0.000015, - output_tokens: 0.000015, - cache_creation_input_tokens: 0.00000375, - input_cache_creation: 0.00000375, - input_cache_creation_5m: 0.00000375, - input_cache_creation_1h: 0.000006, - cache_read_input_tokens: 3e-7, - input_cache_read: 3e-7, - }, - }, - ], - }, - { - modelName: "claude-3-haiku-20240307", - matchPattern: - "(?i)^(anthropic/)?(claude-3-haiku-20240307|anthropic\\.claude-3-haiku-20240307-v1:0|claude-3-haiku@20240307)$", - startDate: "2024-03-14T09:41:18.736Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 2.5e-7, - output: 0.00000125, - }, - }, - ], - }, - { - modelName: "gemini-1.0-pro-latest", - matchPattern: "(?i)^(google(ai)?/)?(gemini-1.0-pro-latest)(@[a-zA-Z0-9]+)?$", - startDate: "2024-04-11T10:27:46.517Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 2.5e-7, - output: 5e-7, - }, - }, - ], - }, - { - modelName: "gemini-1.0-pro", - matchPattern: "(?i)^(google(ai)?/)?(gemini-1.0-pro)(@[a-zA-Z0-9]+)?$", - startDate: "2024-04-11T10:27:46.517Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 1.25e-7, - output: 3.75e-7, - }, - }, - ], - }, - { - modelName: "gemini-1.0-pro-001", - matchPattern: "(?i)^(google(ai)?/)?(gemini-1.0-pro-001)(@[a-zA-Z0-9]+)?$", - startDate: "2024-04-11T10:27:46.517Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 1.25e-7, - output: 3.75e-7, - }, - }, - ], - }, - { - modelName: "gemini-pro", - matchPattern: "(?i)^(google(ai)?/)?(gemini-pro)(@[a-zA-Z0-9]+)?$", - startDate: "2024-04-11T10:27:46.517Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 1.25e-7, - output: 3.75e-7, - }, - }, - ], - }, - { - modelName: "gemini-1.5-pro-latest", - matchPattern: "(?i)^(google(ai)?/)?(gemini-1.5-pro-latest)(@[a-zA-Z0-9]+)?$", - startDate: "2024-04-11T10:27:46.517Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.0000025, - output: 0.0000075, - }, - }, - ], - }, - { - modelName: "gpt-4-turbo-2024-04-09", - matchPattern: "(?i)^(openai/)?(gpt-4-turbo-2024-04-09)$", - startDate: "2024-04-23T10:37:17.092Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00001, - output: 0.00003, - }, - }, - ], - }, - { - modelName: "gpt-4-turbo", - matchPattern: "(?i)^(openai/)?(gpt-4-turbo)$", - startDate: "2024-04-11T21:13:44.989Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00001, - output: 0.00003, - }, - }, - ], - }, - { - modelName: "gpt-4-preview", - matchPattern: "(?i)^(openai/)?(gpt-4-preview)$", - startDate: "2024-04-23T10:37:17.092Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00001, - output: 0.00003, - }, - }, - ], - }, - { - modelName: "claude-3-5-sonnet-20240620", - matchPattern: - "(?i)^(anthropic/)?(claude-3-5-sonnet-20240620|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-sonnet-20240620-v1:0|claude-3-5-sonnet@20240620)$", - startDate: "2024-06-25T11:47:24.475Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000003, - input_tokens: 0.000003, - output: 0.000015, - output_tokens: 0.000015, - cache_creation_input_tokens: 0.00000375, - input_cache_creation: 0.00000375, - input_cache_creation_5m: 0.00000375, - input_cache_creation_1h: 0.000006, - cache_read_input_tokens: 3e-7, - input_cache_read: 3e-7, - }, - }, - ], - }, - { - modelName: "gpt-4o-mini", - matchPattern: "(?i)^(openai/)?(gpt-4o-mini)$", - startDate: "2024-07-18T17:56:09.591Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 1.5e-7, - output: 6e-7, - input_cached_tokens: 7.5e-8, - input_cache_read: 7.5e-8, - }, - }, - ], - }, - { - modelName: "gpt-4o-mini-2024-07-18", - matchPattern: "(?i)^(openai/)?(gpt-4o-mini-2024-07-18)$", - startDate: "2024-07-18T17:56:09.591Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 1.5e-7, - input_cached_tokens: 7.5e-8, - input_cache_read: 7.5e-8, - output: 6e-7, - }, - }, - ], - }, - { - modelName: "gpt-4o-2024-08-06", - matchPattern: "(?i)^(openai/)?(gpt-4o-2024-08-06)$", - startDate: "2024-08-07T11:54:31.298Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.0000025, - input_cached_tokens: 0.00000125, - input_cache_read: 0.00000125, - output: 0.00001, - }, - }, - ], - }, - { - modelName: "o1-preview", - matchPattern: "(?i)^(openai/)?(o1-preview)$", - startDate: "2024-09-13T10:01:35.373Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000015, - input_cached_tokens: 0.0000075, - input_cache_read: 0.0000075, - output: 0.00006, - output_reasoning_tokens: 0.00006, - output_reasoning: 0.00006, - }, - }, - ], - }, - { - modelName: "o1-preview-2024-09-12", - matchPattern: "(?i)^(openai/)?(o1-preview-2024-09-12)$", - startDate: "2024-09-13T10:01:35.373Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000015, - input_cached_tokens: 0.0000075, - input_cache_read: 0.0000075, - output: 0.00006, - output_reasoning_tokens: 0.00006, - output_reasoning: 0.00006, - }, - }, - ], - }, - { - modelName: "o1-mini", - matchPattern: "(?i)^(openai/)?(o1-mini)$", - startDate: "2024-09-13T10:01:35.373Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.0000011, - input_cached_tokens: 5.5e-7, - input_cache_read: 5.5e-7, - output: 0.0000044, - output_reasoning_tokens: 0.0000044, - output_reasoning: 0.0000044, - }, - }, - ], - }, - { - modelName: "o1-mini-2024-09-12", - matchPattern: "(?i)^(openai/)?(o1-mini-2024-09-12)$", - startDate: "2024-09-13T10:01:35.373Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.0000011, - input_cached_tokens: 5.5e-7, - input_cache_read: 5.5e-7, - output: 0.0000044, - output_reasoning_tokens: 0.0000044, - output_reasoning: 0.0000044, - }, - }, - ], - }, - { - modelName: "claude-3.5-sonnet-20241022", - matchPattern: - "(?i)^(anthropic/)?(claude-3-5-sonnet-20241022|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-sonnet-20241022-v2:0|claude-3-5-sonnet-V2@20241022)$", - startDate: "2024-10-22T18:48:01.676Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000003, - input_tokens: 0.000003, - output: 0.000015, - output_tokens: 0.000015, - cache_creation_input_tokens: 0.00000375, - input_cache_creation: 0.00000375, - input_cache_creation_5m: 0.00000375, - input_cache_creation_1h: 0.000006, - cache_read_input_tokens: 3e-7, - input_cache_read: 3e-7, - }, - }, - ], - }, - { - modelName: "claude-3.5-sonnet-latest", - matchPattern: "(?i)^(anthropic/)?(claude-3-5-sonnet-latest)$", - startDate: "2024-10-22T18:48:01.676Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000003, - input_tokens: 0.000003, - output: 0.000015, - output_tokens: 0.000015, - cache_creation_input_tokens: 0.00000375, - input_cache_creation: 0.00000375, - input_cache_creation_5m: 0.00000375, - input_cache_creation_1h: 0.000006, - cache_read_input_tokens: 3e-7, - input_cache_read: 3e-7, - }, - }, - ], - }, - { - modelName: "claude-3-5-haiku-20241022", - matchPattern: - "(?i)^(anthropic/)?(claude-3-5-haiku-20241022|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-haiku-20241022-v1:0|claude-3-5-haiku-V1@20241022)$", - startDate: "2024-11-05T10:30:50.566Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 8e-7, - input_tokens: 8e-7, - output: 0.000004, - output_tokens: 0.000004, - cache_creation_input_tokens: 0.000001, - input_cache_creation: 0.000001, - input_cache_creation_5m: 0.000001, - input_cache_creation_1h: 0.0000016, - cache_read_input_tokens: 8e-8, - input_cache_read: 8e-8, - }, - }, - ], - }, - { - modelName: "claude-3.5-haiku-latest", - matchPattern: "(?i)^(anthropic/)?(claude-3-5-haiku-latest)$", - startDate: "2024-11-05T10:30:50.566Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 8e-7, - input_tokens: 8e-7, - output: 0.000004, - output_tokens: 0.000004, - cache_creation_input_tokens: 0.000001, - input_cache_creation: 0.000001, - input_cache_creation_5m: 0.000001, - input_cache_creation_1h: 0.0000016, - cache_read_input_tokens: 8e-8, - input_cache_read: 8e-8, - }, - }, - ], - }, - { - modelName: "chatgpt-4o-latest", - matchPattern: "(?i)^(chatgpt-4o-latest)$", - startDate: "2024-11-25T12:47:17.504Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000005, - output: 0.000015, - }, - }, - ], - }, - { - modelName: "gpt-4o-2024-11-20", - matchPattern: "(?i)^(openai/)?(gpt-4o-2024-11-20)$", - startDate: "2024-12-03T10:06:12.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.0000025, - input_cached_tokens: 0.00000125, - input_cache_read: 0.00000125, - output: 0.00001, - }, - }, - ], - }, - { - modelName: "gpt-4o-audio-preview", - matchPattern: "(?i)^(openai/)?(gpt-4o-audio-preview)$", - startDate: "2024-12-03T10:19:56.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input_text_tokens: 0.0000025, - output_text_tokens: 0.00001, - input_audio_tokens: 0.0001, - input_audio: 0.0001, - output_audio_tokens: 0.0002, - output_audio: 0.0002, - }, - }, - ], - }, - { - modelName: "gpt-4o-audio-preview-2024-10-01", - matchPattern: "(?i)^(openai/)?(gpt-4o-audio-preview-2024-10-01)$", - startDate: "2024-12-03T10:19:56.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input_text_tokens: 0.0000025, - output_text_tokens: 0.00001, - input_audio_tokens: 0.0001, - input_audio: 0.0001, - output_audio_tokens: 0.0002, - output_audio: 0.0002, - }, - }, - ], - }, - { - modelName: "gpt-4o-realtime-preview", - matchPattern: "(?i)^(openai/)?(gpt-4o-realtime-preview)$", - startDate: "2024-12-03T10:19:56.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input_text_tokens: 0.000005, - input_cached_text_tokens: 0.0000025, - output_text_tokens: 0.00002, - input_audio_tokens: 0.0001, - input_audio: 0.0001, - input_cached_audio_tokens: 0.00002, - output_audio_tokens: 0.0002, - output_audio: 0.0002, - }, - }, - ], - }, - { - modelName: "gpt-4o-realtime-preview-2024-10-01", - matchPattern: "(?i)^(openai/)?(gpt-4o-realtime-preview-2024-10-01)$", - startDate: "2024-12-03T10:19:56.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input_text_tokens: 0.000005, - input_cached_text_tokens: 0.0000025, - output_text_tokens: 0.00002, - input_audio_tokens: 0.0001, - input_audio: 0.0001, - input_cached_audio_tokens: 0.00002, - output_audio_tokens: 0.0002, - output_audio: 0.0002, - }, - }, - ], - }, - { - modelName: "o1", - matchPattern: "(?i)^(openai/)?(o1)$", - startDate: "2025-01-17T00:01:35.373Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000015, - input_cached_tokens: 0.0000075, - input_cache_read: 0.0000075, - output: 0.00006, - output_reasoning_tokens: 0.00006, - output_reasoning: 0.00006, - }, - }, - ], - }, - { - modelName: "o1-2024-12-17", - matchPattern: "(?i)^(openai/)?(o1-2024-12-17)$", - startDate: "2025-01-17T00:01:35.373Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000015, - input_cached_tokens: 0.0000075, - input_cache_read: 0.0000075, - output: 0.00006, - output_reasoning_tokens: 0.00006, - output_reasoning: 0.00006, - }, - }, - ], - }, - { - modelName: "o3-mini", - matchPattern: "(?i)^(openai/)?(o3-mini)$", - startDate: "2025-01-31T20:41:35.373Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.0000011, - input_cached_tokens: 5.5e-7, - input_cache_read: 5.5e-7, - output: 0.0000044, - output_reasoning_tokens: 0.0000044, - output_reasoning: 0.0000044, - }, - }, - ], - }, - { - modelName: "o3-mini-2025-01-31", - matchPattern: "(?i)^(openai/)?(o3-mini-2025-01-31)$", - startDate: "2025-01-31T20:41:35.373Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.0000011, - input_cached_tokens: 5.5e-7, - input_cache_read: 5.5e-7, - output: 0.0000044, - output_reasoning_tokens: 0.0000044, - output_reasoning: 0.0000044, - }, - }, - ], - }, - { - modelName: "gemini-2.0-flash-001", - matchPattern: "(?i)^(google(ai)?/)?(gemini-2.0-flash-001)(@[a-zA-Z0-9]+)?$", - startDate: "2025-02-06T11:11:35.241Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 1e-7, - output: 4e-7, - }, - }, - ], - }, - { - modelName: "gemini-2.0-flash-lite-preview-02-05", - matchPattern: "(?i)^(google(ai)?/)?(gemini-2.0-flash-lite-preview-02-05)(@[a-zA-Z0-9]+)?$", - startDate: "2025-02-06T11:11:35.241Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 7.5e-8, - output: 3e-7, - }, - }, - ], - }, - { - modelName: "claude-3.7-sonnet-20250219", - matchPattern: - "(?i)^(anthropic/)?(claude-3.7-sonnet-20250219|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3.7-sonnet-20250219-v1:0|claude-3-7-sonnet-V1@20250219)$", - startDate: "2025-02-25T09:35:39.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000003, - input_tokens: 0.000003, - output: 0.000015, - output_tokens: 0.000015, - cache_creation_input_tokens: 0.00000375, - input_cache_creation: 0.00000375, - input_cache_creation_5m: 0.00000375, - input_cache_creation_1h: 0.000006, - cache_read_input_tokens: 3e-7, - input_cache_read: 3e-7, - }, - }, - ], - }, - { - modelName: "claude-3.7-sonnet-latest", - matchPattern: "(?i)^(anthropic/)?(claude-3-7-sonnet-latest)$", - startDate: "2025-02-25T09:35:39.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000003, - input_tokens: 0.000003, - output: 0.000015, - output_tokens: 0.000015, - cache_creation_input_tokens: 0.00000375, - input_cache_creation: 0.00000375, - input_cache_creation_5m: 0.00000375, - input_cache_creation_1h: 0.000006, - cache_read_input_tokens: 3e-7, - input_cache_read: 3e-7, - }, - }, - ], - }, - { - modelName: "gpt-4.5-preview", - matchPattern: "(?i)^(openai/)?(gpt-4.5-preview)$", - startDate: "2025-02-27T21:26:54.132Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000075, - input_cached_tokens: 0.0000375, - input_cached_text_tokens: 0.0000375, - input_cache_read: 0.0000375, - output: 0.00015, - }, - }, - ], - }, - { - modelName: "gpt-4.5-preview-2025-02-27", - matchPattern: "(?i)^(openai/)?(gpt-4.5-preview-2025-02-27)$", - startDate: "2025-02-27T21:26:54.132Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000075, - input_cached_tokens: 0.0000375, - input_cached_text_tokens: 0.0000375, - input_cache_read: 0.0000375, - output: 0.00015, - }, - }, - ], - }, - { - modelName: "gpt-4.1", - matchPattern: "(?i)^(openai/)?(gpt-4.1)$", - startDate: "2025-04-15T10:26:54.132Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000002, - input_cached_tokens: 5e-7, - input_cached_text_tokens: 5e-7, - input_cache_read: 5e-7, - output: 0.000008, - }, - }, - ], - }, - { - modelName: "gpt-4.1-2025-04-14", - matchPattern: "(?i)^(openai/)?(gpt-4.1-2025-04-14)$", - startDate: "2025-04-15T10:26:54.132Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000002, - input_cached_tokens: 5e-7, - input_cached_text_tokens: 5e-7, - input_cache_read: 5e-7, - output: 0.000008, - }, - }, - ], - }, - { - modelName: "gpt-4.1-mini-2025-04-14", - matchPattern: "(?i)^(openai/)?(gpt-4.1-mini-2025-04-14)$", - startDate: "2025-04-15T10:26:54.132Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 4e-7, - input_cached_tokens: 1e-7, - input_cached_text_tokens: 1e-7, - input_cache_read: 1e-7, - output: 0.0000016, - }, - }, - ], - }, - { - modelName: "gpt-4.1-nano-2025-04-14", - matchPattern: "(?i)^(openai/)?(gpt-4.1-nano-2025-04-14)$", - startDate: "2025-04-15T10:26:54.132Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 1e-7, - input_cached_tokens: 2.5e-8, - input_cached_text_tokens: 2.5e-8, - input_cache_read: 2.5e-8, - output: 4e-7, - }, - }, - ], - }, - { - modelName: "o3", - matchPattern: "(?i)^(openai/)?(o3)$", - startDate: "2025-04-16T23:26:54.132Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000002, - input_cached_tokens: 5e-7, - input_cache_read: 5e-7, - output: 0.000008, - output_reasoning_tokens: 0.000008, - output_reasoning: 0.000008, - }, - }, - ], - }, - { - modelName: "o3-2025-04-16", - matchPattern: "(?i)^(openai/)?(o3-2025-04-16)$", - startDate: "2025-04-16T23:26:54.132Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000002, - input_cached_tokens: 5e-7, - input_cache_read: 5e-7, - output: 0.000008, - output_reasoning_tokens: 0.000008, - output_reasoning: 0.000008, - }, - }, - ], - }, - { - modelName: "o4-mini", - matchPattern: "(?i)^(o4-mini)$", - startDate: "2025-04-16T23:26:54.132Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.0000011, - input_cached_tokens: 2.75e-7, - input_cache_read: 2.75e-7, - output: 0.0000044, - output_reasoning_tokens: 0.0000044, - output_reasoning: 0.0000044, - }, - }, - ], - }, - { - modelName: "o4-mini-2025-04-16", - matchPattern: "(?i)^(o4-mini-2025-04-16)$", - startDate: "2025-04-16T23:26:54.132Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.0000011, - input_cached_tokens: 2.75e-7, - input_cache_read: 2.75e-7, - output: 0.0000044, - output_reasoning_tokens: 0.0000044, - output_reasoning: 0.0000044, - }, - }, - ], - }, - { - modelName: "gemini-2.0-flash", - matchPattern: "(?i)^(google(ai)?/)?(gemini-2.0-flash)(@[a-zA-Z0-9]+)?$", - startDate: "2025-04-22T10:11:35.241Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 1e-7, - output: 4e-7, - }, - }, - ], - }, - { - modelName: "gemini-2.0-flash-lite-preview", - matchPattern: "(?i)^(google(ai)?/)?(gemini-2.0-flash-lite-preview)(@[a-zA-Z0-9]+)?$", - startDate: "2025-04-22T10:11:35.241Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 7.5e-8, - output: 3e-7, - }, - }, - ], - }, - { - modelName: "gpt-4.1-nano", - matchPattern: "(?i)^(openai/)?(gpt-4.1-nano)$", - startDate: "2025-04-22T10:11:35.241Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 1e-7, - input_cached_tokens: 2.5e-8, - input_cached_text_tokens: 2.5e-8, - input_cache_read: 2.5e-8, - output: 4e-7, - }, - }, - ], - }, - { - modelName: "gpt-4.1-mini", - matchPattern: "(?i)^(openai/)?(gpt-4.1-mini)$", - startDate: "2025-04-22T10:11:35.241Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 4e-7, - input_cached_tokens: 1e-7, - input_cached_text_tokens: 1e-7, - input_cache_read: 1e-7, - output: 0.0000016, - }, - }, - ], - }, - { - modelName: "claude-sonnet-4-5-20250929", - matchPattern: - "(?i)^(anthropic/)?(claude-sonnet-4-5(-20250929)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4-5(-20250929)?-v1(:0)?|claude-sonnet-4-5-V1(@20250929)?|claude-sonnet-4-5(@20250929)?)$", - startDate: "2025-09-29T00:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000003, - input_tokens: 0.000003, - output: 0.000015, - output_tokens: 0.000015, - cache_creation_input_tokens: 0.00000375, - input_cache_creation: 0.00000375, - input_cache_creation_5m: 0.00000375, - input_cache_creation_1h: 0.000006, - cache_read_input_tokens: 3e-7, - input_cache_read: 3e-7, - }, - }, - { - name: "Large Context", - isDefault: false, - priority: 1, - conditions: [ + "modelName": "gpt-4o", + "matchPattern": "(?i)^(openai/)?(gpt-4o)$", + "startDate": "2024-05-13T23:15:07.670Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 0.00000125, + "input_cache_read": 0.00000125, + "output": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-4o-2024-05-13", + "matchPattern": "(?i)^(openai/)?(gpt-4o-2024-05-13)$", + "startDate": "2024-05-13T23:15:07.670Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000005, + "output": 0.000015 + } + } + ] + }, + { + "modelName": "gpt-4-1106-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4-1106-preview)$", + "startDate": "2024-04-23T10:37:17.092Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "gpt-4-turbo-vision", + "matchPattern": "(?i)^(openai/)?(gpt-4(-\\d{4})?-vision-preview)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "gpt-4-32k", + "matchPattern": "(?i)^(openai/)?(gpt-4-32k)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00006, + "output": 0.00012 + } + } + ] + }, + { + "modelName": "gpt-4-32k-0613", + "matchPattern": "(?i)^(openai/)?(gpt-4-32k-0613)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00006, + "output": 0.00012 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-1106", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-1106)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000001, + "output": 0.000002 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-0613", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-0613)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000015, + "output": 0.000002 + } + } + ] + }, + { + "modelName": "gpt-4-0613", + "matchPattern": "(?i)^(openai/)?(gpt-4-0613)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00006 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-instruct", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-instruct)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000015, + "output": 0.000002 + } + } + ] + }, + { + "modelName": "text-ada-001", + "matchPattern": "(?i)^(text-ada-001)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.000004 + } + } + ] + }, + { + "modelName": "text-babbage-001", + "matchPattern": "(?i)^(text-babbage-001)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 5e-7 + } + } + ] + }, + { + "modelName": "text-curie-001", + "matchPattern": "(?i)^(text-curie-001)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "modelName": "text-davinci-001", + "matchPattern": "(?i)^(text-davinci-001)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "modelName": "text-davinci-002", + "matchPattern": "(?i)^(text-davinci-002)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "modelName": "text-davinci-003", + "matchPattern": "(?i)^(text-davinci-003)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "modelName": "text-embedding-ada-002-v2", + "matchPattern": "(?i)^(text-embedding-ada-002-v2)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "modelName": "text-embedding-ada-002", + "matchPattern": "(?i)^(text-embedding-ada-002)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-16k-0613", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-16k-0613)$", + "startDate": "2024-02-03T17:29:57.350Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "output": 0.000004 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-0301", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-0301)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "output": 0.000002 + } + } + ] + }, + { + "modelName": "gpt-4-32k-0314", + "matchPattern": "(?i)^(openai/)?(gpt-4-32k-0314)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00006, + "output": 0.00012 + } + } + ] + }, + { + "modelName": "gpt-4-0314", + "matchPattern": "(?i)^(openai/)?(gpt-4-0314)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00006 + } + } + ] + }, + { + "modelName": "gpt-4", + "matchPattern": "(?i)^(openai/)?(gpt-4)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00006 + } + } + ] + }, + { + "modelName": "claude-instant-1.2", + "matchPattern": "(?i)^(anthropic/)?(claude-instant-1.2)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000163, + "output": 0.00000551 + } + } + ] + }, + { + "modelName": "claude-2.0", + "matchPattern": "(?i)^(anthropic/)?(claude-2.0)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "modelName": "claude-2.1", + "matchPattern": "(?i)^(anthropic/)?(claude-2.1)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "modelName": "claude-1.3", + "matchPattern": "(?i)^(anthropic/)?(claude-1.3)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "modelName": "claude-1.2", + "matchPattern": "(?i)^(anthropic/)?(claude-1.2)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "modelName": "claude-1.1", + "matchPattern": "(?i)^(anthropic/)?(claude-1.1)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "modelName": "claude-instant-1", + "matchPattern": "(?i)^(anthropic/)?(claude-instant-1)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000163, + "output": 0.00000551 + } + } + ] + }, + { + "modelName": "babbage-002", + "matchPattern": "(?i)^(babbage-002)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 4e-7, + "output": 0.0000016 + } + } + ] + }, + { + "modelName": "davinci-002", + "matchPattern": "(?i)^(davinci-002)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000006, + "output": 0.000012 + } + } + ] + }, + { + "modelName": "text-embedding-3-small", + "matchPattern": "(?i)^(text-embedding-3-small)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 2e-8 + } + } + ] + }, + { + "modelName": "text-embedding-3-large", + "matchPattern": "(?i)^(text-embedding-3-large)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1.3e-7 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-0125", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-0125)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "output": 0.0000015 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo)$", + "startDate": "2024-02-13T12:00:37.424Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "output": 0.0000015 + } + } + ] + }, + { + "modelName": "gpt-4-0125-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4-0125-preview)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "ft:gpt-3.5-turbo-1106", + "matchPattern": "(?i)^(ft:)(gpt-3.5-turbo-1106:)(.+)(:)(.*)(:)(.+)$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "output": 0.000006 + } + } + ] + }, + { + "modelName": "ft:gpt-3.5-turbo-0613", + "matchPattern": "(?i)^(ft:)(gpt-3.5-turbo-0613:)(.+)(:)(.*)(:)(.+)$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000012, + "output": 0.000016 + } + } + ] + }, + { + "modelName": "ft:davinci-002", + "matchPattern": "(?i)^(ft:)(davinci-002:)(.+)(:)(.*)(:)(.+)$$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000012, + "output": 0.000012 + } + } + ] + }, + { + "modelName": "ft:babbage-002", + "matchPattern": "(?i)^(ft:)(babbage-002:)(.+)(:)(.*)(:)(.+)$$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000016, + "output": 0.0000016 + } + } + ] + }, + { + "modelName": "chat-bison", + "matchPattern": "(?i)^(chat-bison)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "codechat-bison-32k", + "matchPattern": "(?i)^(codechat-bison-32k)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "codechat-bison", + "matchPattern": "(?i)^(codechat-bison)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "text-bison-32k", + "matchPattern": "(?i)^(text-bison-32k)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "chat-bison-32k", + "matchPattern": "(?i)^(chat-bison-32k)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "text-unicorn", + "matchPattern": "(?i)^(text-unicorn)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "output": 0.0000075 + } + } + ] + }, + { + "modelName": "text-bison", + "matchPattern": "(?i)^(text-bison)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "textembedding-gecko", + "matchPattern": "(?i)^(textembedding-gecko)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "modelName": "textembedding-gecko-multilingual", + "matchPattern": "(?i)^(textembedding-gecko-multilingual)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "modelName": "code-gecko", + "matchPattern": "(?i)^(code-gecko)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "code-bison", + "matchPattern": "(?i)^(code-bison)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "code-bison-32k", + "matchPattern": "(?i)^(code-bison-32k)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-16k", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-16k)$", + "startDate": "2024-02-13T12:00:37.424Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "output": 0.0000015 + } + } + ] + }, + { + "modelName": "gpt-4-turbo-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4-turbo-preview)$", + "startDate": "2024-02-15T21:21:50.947Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "claude-3-opus-20240229", + "matchPattern": "(?i)^(anthropic/)?(claude-3-opus-20240229|anthropic\\.claude-3-opus-20240229-v1:0|claude-3-opus@20240229)$", + "startDate": "2024-03-07T17:55:38.139Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "output": 0.000075 + } + } + ] + }, + { + "modelName": "claude-3-sonnet-20240229", + "matchPattern": "(?i)^(anthropic/)?(claude-3-sonnet-20240229|anthropic\\.claude-3-sonnet-20240229-v1:0|claude-3-sonnet@20240229)$", + "startDate": "2024-03-07T17:55:38.139Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-3-haiku-20240307", + "matchPattern": "(?i)^(anthropic/)?(claude-3-haiku-20240307|anthropic\\.claude-3-haiku-20240307-v1:0|claude-3-haiku@20240307)$", + "startDate": "2024-03-14T09:41:18.736Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 0.00000125 + } + } + ] + }, + { + "modelName": "gemini-1.0-pro-latest", + "matchPattern": "(?i)^(google(ai)?/)?(gemini-1.0-pro-latest)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-04-11T10:27:46.517Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "gemini-1.0-pro", + "matchPattern": "(?i)^(google(ai)?/)?(gemini-1.0-pro)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-04-11T10:27:46.517Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.25e-7, + "output": 3.75e-7 + } + } + ] + }, + { + "modelName": "gemini-1.0-pro-001", + "matchPattern": "(?i)^(google(ai)?/)?(gemini-1.0-pro-001)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-04-11T10:27:46.517Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.25e-7, + "output": 3.75e-7 + } + } + ] + }, + { + "modelName": "gemini-pro", + "matchPattern": "(?i)^(google(ai)?/)?(gemini-pro)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-04-11T10:27:46.517Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.25e-7, + "output": 3.75e-7 + } + } + ] + }, + { + "modelName": "gemini-1.5-pro-latest", + "matchPattern": "(?i)^(google(ai)?/)?(gemini-1.5-pro-latest)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-04-11T10:27:46.517Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "output": 0.0000075 + } + } + ] + }, + { + "modelName": "gpt-4-turbo-2024-04-09", + "matchPattern": "(?i)^(openai/)?(gpt-4-turbo-2024-04-09)$", + "startDate": "2024-04-23T10:37:17.092Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "gpt-4-turbo", + "matchPattern": "(?i)^(openai/)?(gpt-4-turbo)$", + "startDate": "2024-04-11T21:13:44.989Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "gpt-4-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4-preview)$", + "startDate": "2024-04-23T10:37:17.092Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "claude-3-5-sonnet-20240620", + "matchPattern": "(?i)^(anthropic/)?(claude-3-5-sonnet-20240620|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-sonnet-20240620-v1:0|claude-3-5-sonnet@20240620)$", + "startDate": "2024-06-25T11:47:24.475Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "gpt-4o-mini", + "matchPattern": "(?i)^(openai/)?(gpt-4o-mini)$", + "startDate": "2024-07-18T17:56:09.591Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.5e-7, + "output": 6e-7, + "input_cached_tokens": 7.5e-8, + "input_cache_read": 7.5e-8 + } + } + ] + }, + { + "modelName": "gpt-4o-mini-2024-07-18", + "matchPattern": "(?i)^(openai/)?(gpt-4o-mini-2024-07-18)$", + "startDate": "2024-07-18T17:56:09.591Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.5e-7, + "input_cached_tokens": 7.5e-8, + "input_cache_read": 7.5e-8, + "output": 6e-7 + } + } + ] + }, + { + "modelName": "gpt-4o-2024-08-06", + "matchPattern": "(?i)^(openai/)?(gpt-4o-2024-08-06)$", + "startDate": "2024-08-07T11:54:31.298Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 0.00000125, + "input_cache_read": 0.00000125, + "output": 0.00001 + } + } + ] + }, + { + "modelName": "o1-preview", + "matchPattern": "(?i)^(openai/)?(o1-preview)$", + "startDate": "2024-09-13T10:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "modelName": "o1-preview-2024-09-12", + "matchPattern": "(?i)^(openai/)?(o1-preview-2024-09-12)$", + "startDate": "2024-09-13T10:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "modelName": "o1-mini", + "matchPattern": "(?i)^(openai/)?(o1-mini)$", + "startDate": "2024-09-13T10:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "o1-mini-2024-09-12", + "matchPattern": "(?i)^(openai/)?(o1-mini-2024-09-12)$", + "startDate": "2024-09-13T10:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "claude-3.5-sonnet-20241022", + "matchPattern": "(?i)^(anthropic/)?(claude-3-5-sonnet-20241022|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-sonnet-20241022-v2:0|claude-3-5-sonnet-V2@20241022)$", + "startDate": "2024-10-22T18:48:01.676Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-3.5-sonnet-latest", + "matchPattern": "(?i)^(anthropic/)?(claude-3-5-sonnet-latest)$", + "startDate": "2024-10-22T18:48:01.676Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-3-5-haiku-20241022", + "matchPattern": "(?i)^(anthropic/)?(claude-3-5-haiku-20241022|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-haiku-20241022-v1:0|claude-3-5-haiku-V1@20241022)$", + "startDate": "2024-11-05T10:30:50.566Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 8e-7, + "input_tokens": 8e-7, + "output": 0.000004, + "output_tokens": 0.000004, + "cache_creation_input_tokens": 0.000001, + "input_cache_creation": 0.000001, + "input_cache_creation_5m": 0.000001, + "input_cache_creation_1h": 0.0000016, + "cache_read_input_tokens": 8e-8, + "input_cache_read": 8e-8 + } + } + ] + }, + { + "modelName": "claude-3.5-haiku-latest", + "matchPattern": "(?i)^(anthropic/)?(claude-3-5-haiku-latest)$", + "startDate": "2024-11-05T10:30:50.566Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 8e-7, + "input_tokens": 8e-7, + "output": 0.000004, + "output_tokens": 0.000004, + "cache_creation_input_tokens": 0.000001, + "input_cache_creation": 0.000001, + "input_cache_creation_5m": 0.000001, + "input_cache_creation_1h": 0.0000016, + "cache_read_input_tokens": 8e-8, + "input_cache_read": 8e-8 + } + } + ] + }, + { + "modelName": "chatgpt-4o-latest", + "matchPattern": "(?i)^(chatgpt-4o-latest)$", + "startDate": "2024-11-25T12:47:17.504Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000005, + "output": 0.000015 + } + } + ] + }, + { + "modelName": "gpt-4o-2024-11-20", + "matchPattern": "(?i)^(openai/)?(gpt-4o-2024-11-20)$", + "startDate": "2024-12-03T10:06:12.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 0.00000125, + "input_cache_read": 0.00000125, + "output": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-4o-audio-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4o-audio-preview)$", + "startDate": "2024-12-03T10:19:56.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.0000025, + "output_text_tokens": 0.00001, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "modelName": "gpt-4o-audio-preview-2024-10-01", + "matchPattern": "(?i)^(openai/)?(gpt-4o-audio-preview-2024-10-01)$", + "startDate": "2024-12-03T10:19:56.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.0000025, + "output_text_tokens": 0.00001, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "modelName": "gpt-4o-realtime-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4o-realtime-preview)$", + "startDate": "2024-12-03T10:19:56.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.000005, + "input_cached_text_tokens": 0.0000025, + "output_text_tokens": 0.00002, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "input_cached_audio_tokens": 0.00002, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "modelName": "gpt-4o-realtime-preview-2024-10-01", + "matchPattern": "(?i)^(openai/)?(gpt-4o-realtime-preview-2024-10-01)$", + "startDate": "2024-12-03T10:19:56.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.000005, + "input_cached_text_tokens": 0.0000025, + "output_text_tokens": 0.00002, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "input_cached_audio_tokens": 0.00002, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "modelName": "o1", + "matchPattern": "(?i)^(openai/)?(o1)$", + "startDate": "2025-01-17T00:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "modelName": "o1-2024-12-17", + "matchPattern": "(?i)^(openai/)?(o1-2024-12-17)$", + "startDate": "2025-01-17T00:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "modelName": "o3-mini", + "matchPattern": "(?i)^(openai/)?(o3-mini)$", + "startDate": "2025-01-31T20:41:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "o3-mini-2025-01-31", + "matchPattern": "(?i)^(openai/)?(o3-mini-2025-01-31)$", + "startDate": "2025-01-31T20:41:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "gemini-2.0-flash-001", + "matchPattern": "(?i)^(google(ai)?/)?(gemini-2.0-flash-001)(@[a-zA-Z0-9]+)?$", + "startDate": "2025-02-06T11:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "output": 4e-7 + } + } + ] + }, + { + "modelName": "gemini-2.0-flash-lite-preview-02-05", + "matchPattern": "(?i)^(google(ai)?/)?(gemini-2.0-flash-lite-preview-02-05)(@[a-zA-Z0-9]+)?$", + "startDate": "2025-02-06T11:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 7.5e-8, + "output": 3e-7 + } + } + ] + }, + { + "modelName": "claude-3.7-sonnet-20250219", + "matchPattern": "(?i)^(anthropic/)?(claude-3.7-sonnet-20250219|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3.7-sonnet-20250219-v1:0|claude-3-7-sonnet-V1@20250219)$", + "startDate": "2025-02-25T09:35:39.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-3.7-sonnet-latest", + "matchPattern": "(?i)^(anthropic/)?(claude-3-7-sonnet-latest)$", + "startDate": "2025-02-25T09:35:39.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "gpt-4.5-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4.5-preview)$", + "startDate": "2025-02-27T21:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000075, + "input_cached_tokens": 0.0000375, + "input_cached_text_tokens": 0.0000375, + "input_cache_read": 0.0000375, + "output": 0.00015 + } + } + ] + }, + { + "modelName": "gpt-4.5-preview-2025-02-27", + "matchPattern": "(?i)^(openai/)?(gpt-4.5-preview-2025-02-27)$", + "startDate": "2025-02-27T21:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000075, + "input_cached_tokens": 0.0000375, + "input_cached_text_tokens": 0.0000375, + "input_cache_read": 0.0000375, + "output": 0.00015 + } + } + ] + }, + { + "modelName": "gpt-4.1", + "matchPattern": "(?i)^(openai/)?(gpt-4.1)$", + "startDate": "2025-04-15T10:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cached_text_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008 + } + } + ] + }, + { + "modelName": "gpt-4.1-2025-04-14", + "matchPattern": "(?i)^(openai/)?(gpt-4.1-2025-04-14)$", + "startDate": "2025-04-15T10:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cached_text_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008 + } + } + ] + }, + { + "modelName": "gpt-4.1-mini-2025-04-14", + "matchPattern": "(?i)^(openai/)?(gpt-4.1-mini-2025-04-14)$", + "startDate": "2025-04-15T10:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 4e-7, + "input_cached_tokens": 1e-7, + "input_cached_text_tokens": 1e-7, + "input_cache_read": 1e-7, + "output": 0.0000016 + } + } + ] + }, + { + "modelName": "gpt-4.1-nano-2025-04-14", + "matchPattern": "(?i)^(openai/)?(gpt-4.1-nano-2025-04-14)$", + "startDate": "2025-04-15T10:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "input_cached_tokens": 2.5e-8, + "input_cached_text_tokens": 2.5e-8, + "input_cache_read": 2.5e-8, + "output": 4e-7 + } + } + ] + }, + { + "modelName": "o3", + "matchPattern": "(?i)^(openai/)?(o3)$", + "startDate": "2025-04-16T23:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008, + "output_reasoning_tokens": 0.000008, + "output_reasoning": 0.000008 + } + } + ] + }, + { + "modelName": "o3-2025-04-16", + "matchPattern": "(?i)^(openai/)?(o3-2025-04-16)$", + "startDate": "2025-04-16T23:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008, + "output_reasoning_tokens": 0.000008, + "output_reasoning": 0.000008 + } + } + ] + }, + { + "modelName": "o4-mini", + "matchPattern": "(?i)^(o4-mini)$", + "startDate": "2025-04-16T23:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 2.75e-7, + "input_cache_read": 2.75e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "o4-mini-2025-04-16", + "matchPattern": "(?i)^(o4-mini-2025-04-16)$", + "startDate": "2025-04-16T23:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 2.75e-7, + "input_cache_read": 2.75e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "gemini-2.0-flash", + "matchPattern": "(?i)^(google(ai)?/)?(gemini-2.0-flash)(@[a-zA-Z0-9]+)?$", + "startDate": "2025-04-22T10:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "output": 4e-7 + } + } + ] + }, + { + "modelName": "gemini-2.0-flash-lite-preview", + "matchPattern": "(?i)^(google(ai)?/)?(gemini-2.0-flash-lite-preview)(@[a-zA-Z0-9]+)?$", + "startDate": "2025-04-22T10:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 7.5e-8, + "output": 3e-7 + } + } + ] + }, + { + "modelName": "gpt-4.1-nano", + "matchPattern": "(?i)^(openai/)?(gpt-4.1-nano)$", + "startDate": "2025-04-22T10:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "input_cached_tokens": 2.5e-8, + "input_cached_text_tokens": 2.5e-8, + "input_cache_read": 2.5e-8, + "output": 4e-7 + } + } + ] + }, + { + "modelName": "gpt-4.1-mini", + "matchPattern": "(?i)^(openai/)?(gpt-4.1-mini)$", + "startDate": "2025-04-22T10:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 4e-7, + "input_cached_tokens": 1e-7, + "input_cached_text_tokens": 1e-7, + "input_cache_read": 1e-7, + "output": 0.0000016 + } + } + ] + }, + { + "modelName": "claude-sonnet-4-5-20250929", + "matchPattern": "(?i)^(anthropic/)?(claude-sonnet-4-5(-20250929)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4-5(-20250929)?-v1(:0)?|claude-sonnet-4-5-V1(@20250929)?|claude-sonnet-4-5(@20250929)?)$", + "startDate": "2025-09-29T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + }, + { + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ { - usageDetailPattern: "input", - operator: "gt", - value: 200000, - }, + "usageDetailPattern": "input", + "operator": "gt", + "value": 200000 + } ], - prices: { - input: 0.000006, - input_tokens: 0.000006, - output: 0.0000225, - output_tokens: 0.0000225, - cache_creation_input_tokens: 0.0000075, - input_cache_creation: 0.0000075, - input_cache_creation_5m: 0.0000075, - input_cache_creation_1h: 0.000012, - cache_read_input_tokens: 6e-7, - input_cache_read: 6e-7, - }, - }, - ], - }, - { - modelName: "claude-sonnet-4-20250514", - matchPattern: - "(?i)^(anthropic/)?(claude-sonnet-4(-20250514)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4(-20250514)?-v1(:0)?|claude-sonnet-4-V1(@20250514)?|claude-sonnet-4(@20250514)?)$", - startDate: "2025-05-22T17:09:02.131Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000003, - input_tokens: 0.000003, - output: 0.000015, - output_tokens: 0.000015, - cache_creation_input_tokens: 0.00000375, - input_cache_creation: 0.00000375, - input_cache_creation_5m: 0.00000375, - input_cache_creation_1h: 0.000006, - cache_read_input_tokens: 3e-7, - input_cache_read: 3e-7, - }, - }, - ], - }, - { - modelName: "claude-sonnet-4-latest", - matchPattern: "(?i)^(anthropic/)?(claude-sonnet-4-latest)$", - startDate: "2025-05-22T17:09:02.131Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000003, - input_tokens: 0.000003, - output: 0.000015, - output_tokens: 0.000015, - cache_creation_input_tokens: 0.00000375, - input_cache_creation: 0.00000375, - input_cache_creation_5m: 0.00000375, - input_cache_creation_1h: 0.000006, - cache_read_input_tokens: 3e-7, - input_cache_read: 3e-7, - }, - }, - ], - }, - { - modelName: "claude-opus-4-20250514", - matchPattern: - "(?i)^(anthropic/)?(claude-opus-4(-20250514)?|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-opus-4(-20250514)?-v1(:0)?|claude-opus-4(@20250514)?)$", - startDate: "2025-05-22T17:09:02.131Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000015, - input_tokens: 0.000015, - output: 0.000075, - output_tokens: 0.000075, - cache_creation_input_tokens: 0.00001875, - input_cache_creation: 0.00001875, - input_cache_creation_5m: 0.00001875, - input_cache_creation_1h: 0.00003, - cache_read_input_tokens: 0.0000015, - input_cache_read: 0.0000015, - }, - }, - ], - }, - { - modelName: "o3-pro", - matchPattern: "(?i)^(openai/)?(o3-pro)$", - startDate: "2025-06-10T22:26:54.132Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00002, - output: 0.00008, - output_reasoning_tokens: 0.00008, - output_reasoning: 0.00008, - }, - }, - ], - }, - { - modelName: "o3-pro-2025-06-10", - matchPattern: "(?i)^(openai/)?(o3-pro-2025-06-10)$", - startDate: "2025-06-10T22:26:54.132Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00002, - output: 0.00008, - output_reasoning_tokens: 0.00008, - output_reasoning: 0.00008, - }, - }, - ], - }, - { - modelName: "o1-pro", - matchPattern: "(?i)^(openai/)?(o1-pro)$", - startDate: "2025-06-10T22:26:54.132Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00015, - output: 0.0006, - output_reasoning_tokens: 0.0006, - output_reasoning: 0.0006, - }, - }, - ], - }, - { - modelName: "o1-pro-2025-03-19", - matchPattern: "(?i)^(openai/)?(o1-pro-2025-03-19)$", - startDate: "2025-06-10T22:26:54.132Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00015, - output: 0.0006, - output_reasoning_tokens: 0.0006, - output_reasoning: 0.0006, - }, - }, - ], - }, - { - modelName: "gemini-2.5-flash", - matchPattern: "(?i)^(google(ai)?/)?(gemini-2.5-flash)$", - startDate: "2025-07-03T13:44:06.964Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 3e-7, - input_text: 3e-7, - input_modality_1: 3e-7, - prompt_token_count: 3e-7, - promptTokenCount: 3e-7, - input_cached_tokens: 3e-8, - cached_content_token_count: 3e-8, - output: 0.0000025, - output_text: 0.0000025, - output_modality_1: 0.0000025, - candidates_token_count: 0.0000025, - candidatesTokenCount: 0.0000025, - thoughtsTokenCount: 0.0000025, - thoughts_token_count: 0.0000025, - output_reasoning: 0.0000025, - input_audio_tokens: 0.000001, - }, - }, - ], - }, - { - modelName: "gemini-2.5-flash-lite", - matchPattern: "(?i)^(google(ai)?/)?(gemini-2.5-flash-lite)$", - startDate: "2025-07-03T13:44:06.964Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 1e-7, - input_text: 1e-7, - input_modality_1: 1e-7, - prompt_token_count: 1e-7, - promptTokenCount: 1e-7, - input_cached_tokens: 2.5e-8, - cached_content_token_count: 2.5e-8, - output: 4e-7, - output_text: 4e-7, - output_modality_1: 4e-7, - candidates_token_count: 4e-7, - candidatesTokenCount: 4e-7, - thoughtsTokenCount: 4e-7, - thoughts_token_count: 4e-7, - output_reasoning: 4e-7, - input_audio_tokens: 5e-7, - }, - }, - ], - }, - { - modelName: "claude-opus-4-1-20250805", - matchPattern: - "(?i)^(anthropic/)?(claude-opus-4-1(-20250805)?|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-opus-4-1(-20250805)?-v1(:0)?|claude-opus-4-1(@20250805)?)$", - startDate: "2025-08-05T15:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000015, - input_tokens: 0.000015, - output: 0.000075, - output_tokens: 0.000075, - cache_creation_input_tokens: 0.00001875, - input_cache_creation: 0.00001875, - input_cache_creation_5m: 0.00001875, - input_cache_creation_1h: 0.00003, - cache_read_input_tokens: 0.0000015, - input_cache_read: 0.0000015, - }, - }, - ], - }, - { - modelName: "gpt-5", - matchPattern: "(?i)^(openai/)?(gpt-5)$", - startDate: "2025-08-07T16:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00000125, - input_cached_tokens: 1.25e-7, - output: 0.00001, - input_cache_read: 1.25e-7, - output_reasoning_tokens: 0.00001, - output_reasoning: 0.00001, - }, - }, - ], - }, - { - modelName: "gpt-5-2025-08-07", - matchPattern: "(?i)^(openai/)?(gpt-5-2025-08-07)$", - startDate: "2025-08-11T08:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00000125, - input_cached_tokens: 1.25e-7, - output: 0.00001, - input_cache_read: 1.25e-7, - output_reasoning_tokens: 0.00001, - output_reasoning: 0.00001, - }, - }, - ], - }, - { - modelName: "gpt-5-mini", - matchPattern: "(?i)^(openai/)?(gpt-5-mini)$", - startDate: "2025-08-07T16:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 2.5e-7, - input_cached_tokens: 2.5e-8, - output: 0.000002, - input_cache_read: 2.5e-8, - output_reasoning_tokens: 0.000002, - output_reasoning: 0.000002, - }, - }, - ], - }, - { - modelName: "gpt-5-mini-2025-08-07", - matchPattern: "(?i)^(openai/)?(gpt-5-mini-2025-08-07)$", - startDate: "2025-08-11T08:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 2.5e-7, - input_cached_tokens: 2.5e-8, - output: 0.000002, - input_cache_read: 2.5e-8, - output_reasoning_tokens: 0.000002, - output_reasoning: 0.000002, - }, - }, - ], - }, - { - modelName: "gpt-5-nano", - matchPattern: "(?i)^(openai/)?(gpt-5-nano)$", - startDate: "2025-08-07T16:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 5e-8, - input_cached_tokens: 5e-9, - output: 4e-7, - input_cache_read: 5e-9, - output_reasoning_tokens: 4e-7, - output_reasoning: 4e-7, - }, - }, - ], - }, - { - modelName: "gpt-5-nano-2025-08-07", - matchPattern: "(?i)^(openai/)?(gpt-5-nano-2025-08-07)$", - startDate: "2025-08-11T08:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 5e-8, - input_cached_tokens: 5e-9, - output: 4e-7, - input_cache_read: 5e-9, - output_reasoning_tokens: 4e-7, - output_reasoning: 4e-7, - }, - }, - ], - }, - { - modelName: "gpt-5-chat-latest", - matchPattern: "(?i)^(openai/)?(gpt-5-chat-latest)$", - startDate: "2025-08-07T16:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00000125, - input_cached_tokens: 1.25e-7, - output: 0.00001, - input_cache_read: 1.25e-7, - output_reasoning_tokens: 0.00001, - output_reasoning: 0.00001, - }, - }, - ], - }, - { - modelName: "gpt-5-pro", - matchPattern: "(?i)^(openai/)?(gpt-5-pro)$", - startDate: "2025-10-07T08:03:54.727Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000015, - output: 0.00012, - output_reasoning_tokens: 0.00012, - output_reasoning: 0.00012, - }, - }, - ], - }, - { - modelName: "gpt-5-pro-2025-10-06", - matchPattern: "(?i)^(openai/)?(gpt-5-pro-2025-10-06)$", - startDate: "2025-10-07T08:03:54.727Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000015, - output: 0.00012, - output_reasoning_tokens: 0.00012, - output_reasoning: 0.00012, - }, - }, - ], - }, - { - modelName: "claude-haiku-4-5-20251001", - matchPattern: - "(?i)^(anthropic/)?(claude-haiku-4-5-20251001|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-haiku-4-5-20251001-v1:0|claude-4-5-haiku@20251001)$", - startDate: "2025-10-16T08:20:44.558Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000001, - input_tokens: 0.000001, - output: 0.000005, - output_tokens: 0.000005, - cache_creation_input_tokens: 0.00000125, - input_cache_creation: 0.00000125, - input_cache_creation_5m: 0.00000125, - input_cache_creation_1h: 0.000002, - cache_read_input_tokens: 1e-7, - input_cache_read: 1e-7, - }, - }, - ], - }, - { - modelName: "gpt-5.1", - matchPattern: "(?i)^(openai/)?(gpt-5.1)$", - startDate: "2025-11-14T08:57:23.481Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00000125, - input_cached_tokens: 1.25e-7, - output: 0.00001, - input_cache_read: 1.25e-7, - output_reasoning_tokens: 0.00001, - output_reasoning: 0.00001, - }, - }, - ], - }, - { - modelName: "gpt-5.1-2025-11-13", - matchPattern: "(?i)^(openai/)?(gpt-5.1-2025-11-13)$", - startDate: "2025-11-14T08:57:23.481Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00000125, - input_cached_tokens: 1.25e-7, - output: 0.00001, - input_cache_read: 1.25e-7, - output_reasoning_tokens: 0.00001, - output_reasoning: 0.00001, - }, - }, - ], - }, - { - modelName: "claude-opus-4-5-20251101", - matchPattern: - "(?i)^(anthropic/)?(claude-opus-4-5(-20251101)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-opus-4-5(-20251101)?-v1(:0)?|claude-opus-4-5(@20251101)?)$", - startDate: "2025-11-24T20:53:27.571Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000005, - input_tokens: 0.000005, - output: 0.000025, - output_tokens: 0.000025, - cache_creation_input_tokens: 0.00000625, - input_cache_creation: 0.00000625, - input_cache_creation_5m: 0.00000625, - input_cache_creation_1h: 0.00001, - cache_read_input_tokens: 5e-7, - input_cache_read: 5e-7, - }, - }, - ], - }, - { - modelName: "claude-sonnet-4-6", - matchPattern: - "(?i)^(anthropic\\/)?(claude-sonnet-4-6|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4-6(-v1(:0)?)?|claude-sonnet-4-6)$", - startDate: "2026-02-18T00:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000003, - input_tokens: 0.000003, - output: 0.000015, - output_tokens: 0.000015, - cache_creation_input_tokens: 0.00000375, - input_cache_creation: 0.00000375, - input_cache_creation_5m: 0.00000375, - input_cache_creation_1h: 0.000006, - cache_read_input_tokens: 3e-7, - input_cache_read: 3e-7, - }, - }, - { - name: "Large Context", - isDefault: false, - priority: 1, - conditions: [ + "prices": { + "input": 0.000006, + "input_tokens": 0.000006, + "output": 0.0000225, + "output_tokens": 0.0000225, + "cache_creation_input_tokens": 0.0000075, + "input_cache_creation": 0.0000075, + "input_cache_creation_5m": 0.0000075, + "input_cache_creation_1h": 0.000012, + "cache_read_input_tokens": 6e-7, + "input_cache_read": 6e-7 + } + } + ] + }, + { + "modelName": "claude-sonnet-4-20250514", + "matchPattern": "(?i)^(anthropic/)?(claude-sonnet-4(-20250514)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4(-20250514)?-v1(:0)?|claude-sonnet-4-V1(@20250514)?|claude-sonnet-4(@20250514)?)$", + "startDate": "2025-05-22T17:09:02.131Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-sonnet-4-latest", + "matchPattern": "(?i)^(anthropic/)?(claude-sonnet-4-latest)$", + "startDate": "2025-05-22T17:09:02.131Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-opus-4-20250514", + "matchPattern": "(?i)^(anthropic/)?(claude-opus-4(-20250514)?|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-opus-4(-20250514)?-v1(:0)?|claude-opus-4(@20250514)?)$", + "startDate": "2025-05-22T17:09:02.131Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_tokens": 0.000015, + "output": 0.000075, + "output_tokens": 0.000075, + "cache_creation_input_tokens": 0.00001875, + "input_cache_creation": 0.00001875, + "input_cache_creation_5m": 0.00001875, + "input_cache_creation_1h": 0.00003, + "cache_read_input_tokens": 0.0000015, + "input_cache_read": 0.0000015 + } + } + ] + }, + { + "modelName": "o3-pro", + "matchPattern": "(?i)^(openai/)?(o3-pro)$", + "startDate": "2025-06-10T22:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00002, + "output": 0.00008, + "output_reasoning_tokens": 0.00008, + "output_reasoning": 0.00008 + } + } + ] + }, + { + "modelName": "o3-pro-2025-06-10", + "matchPattern": "(?i)^(openai/)?(o3-pro-2025-06-10)$", + "startDate": "2025-06-10T22:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00002, + "output": 0.00008, + "output_reasoning_tokens": 0.00008, + "output_reasoning": 0.00008 + } + } + ] + }, + { + "modelName": "o1-pro", + "matchPattern": "(?i)^(openai/)?(o1-pro)$", + "startDate": "2025-06-10T22:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00015, + "output": 0.0006, + "output_reasoning_tokens": 0.0006, + "output_reasoning": 0.0006 + } + } + ] + }, + { + "modelName": "o1-pro-2025-03-19", + "matchPattern": "(?i)^(openai/)?(o1-pro-2025-03-19)$", + "startDate": "2025-06-10T22:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00015, + "output": 0.0006, + "output_reasoning_tokens": 0.0006, + "output_reasoning": 0.0006 + } + } + ] + }, + { + "modelName": "gemini-2.5-flash", + "matchPattern": "(?i)^(google(ai)?/)?(gemini-2.5-flash)$", + "startDate": "2025-07-03T13:44:06.964Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 3e-7, + "input_text": 3e-7, + "input_modality_1": 3e-7, + "prompt_token_count": 3e-7, + "promptTokenCount": 3e-7, + "input_cached_tokens": 3e-8, + "cached_content_token_count": 3e-8, + "output": 0.0000025, + "output_text": 0.0000025, + "output_modality_1": 0.0000025, + "candidates_token_count": 0.0000025, + "candidatesTokenCount": 0.0000025, + "thoughtsTokenCount": 0.0000025, + "thoughts_token_count": 0.0000025, + "output_reasoning": 0.0000025, + "input_audio_tokens": 0.000001 + } + } + ] + }, + { + "modelName": "gemini-2.5-flash-lite", + "matchPattern": "(?i)^(google(ai)?/)?(gemini-2.5-flash-lite)$", + "startDate": "2025-07-03T13:44:06.964Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "input_text": 1e-7, + "input_modality_1": 1e-7, + "prompt_token_count": 1e-7, + "promptTokenCount": 1e-7, + "input_cached_tokens": 2.5e-8, + "cached_content_token_count": 2.5e-8, + "output": 4e-7, + "output_text": 4e-7, + "output_modality_1": 4e-7, + "candidates_token_count": 4e-7, + "candidatesTokenCount": 4e-7, + "thoughtsTokenCount": 4e-7, + "thoughts_token_count": 4e-7, + "output_reasoning": 4e-7, + "input_audio_tokens": 5e-7 + } + } + ] + }, + { + "modelName": "claude-opus-4-1-20250805", + "matchPattern": "(?i)^(anthropic/)?(claude-opus-4-1(-20250805)?|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-opus-4-1(-20250805)?-v1(:0)?|claude-opus-4-1(@20250805)?)$", + "startDate": "2025-08-05T15:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_tokens": 0.000015, + "output": 0.000075, + "output_tokens": 0.000075, + "cache_creation_input_tokens": 0.00001875, + "input_cache_creation": 0.00001875, + "input_cache_creation_5m": 0.00001875, + "input_cache_creation_1h": 0.00003, + "cache_read_input_tokens": 0.0000015, + "input_cache_read": 0.0000015 + } + } + ] + }, + { + "modelName": "gpt-5", + "matchPattern": "(?i)^(openai/)?(gpt-5)$", + "startDate": "2025-08-07T16:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-5-2025-08-07", + "matchPattern": "(?i)^(openai/)?(gpt-5-2025-08-07)$", + "startDate": "2025-08-11T08:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-5-mini", + "matchPattern": "(?i)^(openai/)?(gpt-5-mini)$", + "startDate": "2025-08-07T16:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "input_cached_tokens": 2.5e-8, + "output": 0.000002, + "input_cache_read": 2.5e-8, + "output_reasoning_tokens": 0.000002, + "output_reasoning": 0.000002 + } + } + ] + }, + { + "modelName": "gpt-5-mini-2025-08-07", + "matchPattern": "(?i)^(openai/)?(gpt-5-mini-2025-08-07)$", + "startDate": "2025-08-11T08:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "input_cached_tokens": 2.5e-8, + "output": 0.000002, + "input_cache_read": 2.5e-8, + "output_reasoning_tokens": 0.000002, + "output_reasoning": 0.000002 + } + } + ] + }, + { + "modelName": "gpt-5-nano", + "matchPattern": "(?i)^(openai/)?(gpt-5-nano)$", + "startDate": "2025-08-07T16:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-8, + "input_cached_tokens": 5e-9, + "output": 4e-7, + "input_cache_read": 5e-9, + "output_reasoning_tokens": 4e-7, + "output_reasoning": 4e-7 + } + } + ] + }, + { + "modelName": "gpt-5-nano-2025-08-07", + "matchPattern": "(?i)^(openai/)?(gpt-5-nano-2025-08-07)$", + "startDate": "2025-08-11T08:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-8, + "input_cached_tokens": 5e-9, + "output": 4e-7, + "input_cache_read": 5e-9, + "output_reasoning_tokens": 4e-7, + "output_reasoning": 4e-7 + } + } + ] + }, + { + "modelName": "gpt-5-chat-latest", + "matchPattern": "(?i)^(openai/)?(gpt-5-chat-latest)$", + "startDate": "2025-08-07T16:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-5-pro", + "matchPattern": "(?i)^(openai/)?(gpt-5-pro)$", + "startDate": "2025-10-07T08:03:54.727Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "output": 0.00012, + "output_reasoning_tokens": 0.00012, + "output_reasoning": 0.00012 + } + } + ] + }, + { + "modelName": "gpt-5-pro-2025-10-06", + "matchPattern": "(?i)^(openai/)?(gpt-5-pro-2025-10-06)$", + "startDate": "2025-10-07T08:03:54.727Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "output": 0.00012, + "output_reasoning_tokens": 0.00012, + "output_reasoning": 0.00012 + } + } + ] + }, + { + "modelName": "claude-haiku-4-5-20251001", + "matchPattern": "(?i)^(anthropic/)?(claude-haiku-4-5-20251001|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-haiku-4-5-20251001-v1:0|claude-4-5-haiku@20251001)$", + "startDate": "2025-10-16T08:20:44.558Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000001, + "input_tokens": 0.000001, + "output": 0.000005, + "output_tokens": 0.000005, + "cache_creation_input_tokens": 0.00000125, + "input_cache_creation": 0.00000125, + "input_cache_creation_5m": 0.00000125, + "input_cache_creation_1h": 0.000002, + "cache_read_input_tokens": 1e-7, + "input_cache_read": 1e-7 + } + } + ] + }, + { + "modelName": "gpt-5.1", + "matchPattern": "(?i)^(openai/)?(gpt-5.1)$", + "startDate": "2025-11-14T08:57:23.481Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-5.1-2025-11-13", + "matchPattern": "(?i)^(openai/)?(gpt-5.1-2025-11-13)$", + "startDate": "2025-11-14T08:57:23.481Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "modelName": "claude-opus-4-5-20251101", + "matchPattern": "(?i)^(anthropic/)?(claude-opus-4-5(-20251101)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-opus-4-5(-20251101)?-v1(:0)?|claude-opus-4-5(@20251101)?)$", + "startDate": "2025-11-24T20:53:27.571Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000005, + "input_tokens": 0.000005, + "output": 0.000025, + "output_tokens": 0.000025, + "cache_creation_input_tokens": 0.00000625, + "input_cache_creation": 0.00000625, + "input_cache_creation_5m": 0.00000625, + "input_cache_creation_1h": 0.00001, + "cache_read_input_tokens": 5e-7, + "input_cache_read": 5e-7 + } + } + ] + }, + { + "modelName": "claude-sonnet-4-6", + "matchPattern": "(?i)^(anthropic\\/)?(claude-sonnet-4-6|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4-6(-v1(:0)?)?|claude-sonnet-4-6)$", + "startDate": "2026-02-18T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + }, + { + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ { - usageDetailPattern: "input", - operator: "gt", - value: 200000, - }, + "usageDetailPattern": "input", + "operator": "gt", + "value": 200000 + } ], - prices: { - input: 0.000006, - input_tokens: 0.000006, - output: 0.0000225, - output_tokens: 0.0000225, - cache_creation_input_tokens: 0.0000075, - input_cache_creation: 0.0000075, - input_cache_creation_5m: 0.0000075, - input_cache_creation_1h: 0.000012, - cache_read_input_tokens: 6e-7, - input_cache_read: 6e-7, - }, - }, - ], - }, - { - modelName: "claude-opus-4-6", - matchPattern: - "(?i)^(anthropic/)?(claude-opus-4-6|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-opus-4-6-v1(:0)?|claude-opus-4-6)$", - startDate: "2026-02-09T00:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000005, - input_tokens: 0.000005, - output: 0.000025, - output_tokens: 0.000025, - cache_creation_input_tokens: 0.00000625, - input_cache_creation: 0.00000625, - input_cache_creation_5m: 0.00000625, - input_cache_creation_1h: 0.00001, - cache_read_input_tokens: 5e-7, - input_cache_read: 5e-7, - }, - }, - ], - }, - { - modelName: "gemini-2.5-pro", - matchPattern: "(?i)^(google(ai)?/)?(gemini-2.5-pro)$", - startDate: "2025-11-26T13:27:53.545Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00000125, - input_text: 0.00000125, - input_modality_1: 0.00000125, - prompt_token_count: 0.00000125, - promptTokenCount: 0.00000125, - input_cached_tokens: 1.25e-7, - cached_content_token_count: 1.25e-7, - output: 0.00001, - output_text: 0.00001, - output_modality_1: 0.00001, - candidates_token_count: 0.00001, - candidatesTokenCount: 0.00001, - thoughtsTokenCount: 0.00001, - thoughts_token_count: 0.00001, - output_reasoning: 0.00001, - }, - }, - { - name: "Large Context", - isDefault: false, - priority: 1, - conditions: [ + "prices": { + "input": 0.000006, + "input_tokens": 0.000006, + "output": 0.0000225, + "output_tokens": 0.0000225, + "cache_creation_input_tokens": 0.0000075, + "input_cache_creation": 0.0000075, + "input_cache_creation_5m": 0.0000075, + "input_cache_creation_1h": 0.000012, + "cache_read_input_tokens": 6e-7, + "input_cache_read": 6e-7 + } + } + ] + }, + { + "modelName": "claude-opus-4-6", + "matchPattern": "(?i)^(anthropic/)?(claude-opus-4-6|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-opus-4-6-v1(:0)?|claude-opus-4-6)$", + "startDate": "2026-02-09T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000005, + "input_tokens": 0.000005, + "output": 0.000025, + "output_tokens": 0.000025, + "cache_creation_input_tokens": 0.00000625, + "input_cache_creation": 0.00000625, + "input_cache_creation_5m": 0.00000625, + "input_cache_creation_1h": 0.00001, + "cache_read_input_tokens": 5e-7, + "input_cache_read": 5e-7 + } + } + ] + }, + { + "modelName": "gemini-2.5-pro", + "matchPattern": "(?i)^(google(ai)?/)?(gemini-2.5-pro)$", + "startDate": "2025-11-26T13:27:53.545Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_text": 0.00000125, + "input_modality_1": 0.00000125, + "prompt_token_count": 0.00000125, + "promptTokenCount": 0.00000125, + "input_cached_tokens": 1.25e-7, + "cached_content_token_count": 1.25e-7, + "output": 0.00001, + "output_text": 0.00001, + "output_modality_1": 0.00001, + "candidates_token_count": 0.00001, + "candidatesTokenCount": 0.00001, + "thoughtsTokenCount": 0.00001, + "thoughts_token_count": 0.00001, + "output_reasoning": 0.00001 + } + }, + { + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ { - usageDetailPattern: "(input|prompt|cached)", - operator: "gt", - value: 200000, - }, + "usageDetailPattern": "(input|prompt|cached)", + "operator": "gt", + "value": 200000 + } ], - prices: { - input: 0.0000025, - input_text: 0.0000025, - input_modality_1: 0.0000025, - prompt_token_count: 0.0000025, - promptTokenCount: 0.0000025, - input_cached_tokens: 2.5e-7, - cached_content_token_count: 2.5e-7, - output: 0.000015, - output_text: 0.000015, - output_modality_1: 0.000015, - candidates_token_count: 0.000015, - candidatesTokenCount: 0.000015, - thoughtsTokenCount: 0.000015, - thoughts_token_count: 0.000015, - output_reasoning: 0.000015, - }, - }, - ], - }, - { - modelName: "gemini-3-pro-preview", - matchPattern: "(?i)^(google(ai)?/)?(gemini-3-pro-preview)$", - startDate: "2025-11-26T13:27:53.545Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000002, - input_text: 0.000002, - input_modality_1: 0.000002, - prompt_token_count: 0.000002, - promptTokenCount: 0.000002, - input_cached_tokens: 2e-7, - cached_content_token_count: 2e-7, - output: 0.000012, - output_text: 0.000012, - output_modality_1: 0.000012, - candidates_token_count: 0.000012, - candidatesTokenCount: 0.000012, - thoughtsTokenCount: 0.000012, - thoughts_token_count: 0.000012, - output_reasoning: 0.000012, - }, - }, - { - name: "Large Context", - isDefault: false, - priority: 1, - conditions: [ + "prices": { + "input": 0.0000025, + "input_text": 0.0000025, + "input_modality_1": 0.0000025, + "prompt_token_count": 0.0000025, + "promptTokenCount": 0.0000025, + "input_cached_tokens": 2.5e-7, + "cached_content_token_count": 2.5e-7, + "output": 0.000015, + "output_text": 0.000015, + "output_modality_1": 0.000015, + "candidates_token_count": 0.000015, + "candidatesTokenCount": 0.000015, + "thoughtsTokenCount": 0.000015, + "thoughts_token_count": 0.000015, + "output_reasoning": 0.000015 + } + } + ] + }, + { + "modelName": "gemini-3-pro-preview", + "matchPattern": "(?i)^(google(ai)?/)?(gemini-3-pro-preview)$", + "startDate": "2025-11-26T13:27:53.545Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_text": 0.000002, + "input_modality_1": 0.000002, + "prompt_token_count": 0.000002, + "promptTokenCount": 0.000002, + "input_cached_tokens": 2e-7, + "cached_content_token_count": 2e-7, + "output": 0.000012, + "output_text": 0.000012, + "output_modality_1": 0.000012, + "candidates_token_count": 0.000012, + "candidatesTokenCount": 0.000012, + "thoughtsTokenCount": 0.000012, + "thoughts_token_count": 0.000012, + "output_reasoning": 0.000012 + } + }, + { + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ { - usageDetailPattern: "(input|prompt|cached)", - operator: "gt", - value: 200000, - }, + "usageDetailPattern": "(input|prompt|cached)", + "operator": "gt", + "value": 200000 + } ], - prices: { - input: 0.000004, - input_text: 0.000004, - input_modality_1: 0.000004, - prompt_token_count: 0.000004, - promptTokenCount: 0.000004, - input_cached_tokens: 4e-7, - cached_content_token_count: 4e-7, - output: 0.000018, - output_text: 0.000018, - output_modality_1: 0.000018, - candidates_token_count: 0.000018, - candidatesTokenCount: 0.000018, - thoughtsTokenCount: 0.000018, - thoughts_token_count: 0.000018, - output_reasoning: 0.000018, - }, - }, - ], - }, - { - modelName: "gemini-3.1-pro-preview", - matchPattern: "(?i)^(google(ai)?/)?(gemini-3.1-pro-preview(-customtools)?)$", - startDate: "2026-02-19T00:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000002, - input_modality_1: 0.000002, - input_text: 0.000002, - prompt_token_count: 0.000002, - promptTokenCount: 0.000002, - input_cached_tokens: 2e-7, - cached_content_token_count: 2e-7, - output: 0.000012, - output_text: 0.000012, - output_modality_1: 0.000012, - candidates_token_count: 0.000012, - candidatesTokenCount: 0.000012, - thoughtsTokenCount: 0.000012, - thoughts_token_count: 0.000012, - output_reasoning: 0.000012, - }, - }, - { - name: "Large Context", - isDefault: false, - priority: 1, - conditions: [ + "prices": { + "input": 0.000004, + "input_text": 0.000004, + "input_modality_1": 0.000004, + "prompt_token_count": 0.000004, + "promptTokenCount": 0.000004, + "input_cached_tokens": 4e-7, + "cached_content_token_count": 4e-7, + "output": 0.000018, + "output_text": 0.000018, + "output_modality_1": 0.000018, + "candidates_token_count": 0.000018, + "candidatesTokenCount": 0.000018, + "thoughtsTokenCount": 0.000018, + "thoughts_token_count": 0.000018, + "output_reasoning": 0.000018 + } + } + ] + }, + { + "modelName": "gemini-3.1-pro-preview", + "matchPattern": "(?i)^(google(ai)?/)?(gemini-3.1-pro-preview(-customtools)?)$", + "startDate": "2026-02-19T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_modality_1": 0.000002, + "input_text": 0.000002, + "prompt_token_count": 0.000002, + "promptTokenCount": 0.000002, + "input_cached_tokens": 2e-7, + "cached_content_token_count": 2e-7, + "output": 0.000012, + "output_text": 0.000012, + "output_modality_1": 0.000012, + "candidates_token_count": 0.000012, + "candidatesTokenCount": 0.000012, + "thoughtsTokenCount": 0.000012, + "thoughts_token_count": 0.000012, + "output_reasoning": 0.000012 + } + }, + { + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ { - usageDetailPattern: "(input|prompt|cached)", - operator: "gt", - value: 200000, - }, + "usageDetailPattern": "(input|prompt|cached)", + "operator": "gt", + "value": 200000 + } ], - prices: { - input: 0.000004, - input_modality_1: 0.000004, - input_text: 0.000004, - prompt_token_count: 0.000004, - promptTokenCount: 0.000004, - input_cached_tokens: 4e-7, - cached_content_token_count: 4e-7, - output: 0.000018, - output_text: 0.000018, - output_modality_1: 0.000018, - candidates_token_count: 0.000018, - candidatesTokenCount: 0.000018, - thoughtsTokenCount: 0.000018, - thoughts_token_count: 0.000018, - output_reasoning: 0.000018, - }, - }, - ], - }, - { - modelName: "gpt-5.2", - matchPattern: "(?i)^(openai/)?(gpt-5.2)$", - startDate: "2025-12-12T09:00:06.513Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00000175, - input_cached_tokens: 1.75e-7, - input_cache_read: 1.75e-7, - output: 0.000014, - output_reasoning_tokens: 0.000014, - output_reasoning: 0.000014, - }, - }, - ], - }, - { - modelName: "gpt-5.2-2025-12-11", - matchPattern: "(?i)^(openai/)?(gpt-5.2-2025-12-11)$", - startDate: "2025-12-12T09:00:06.513Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00000175, - input_cached_tokens: 1.75e-7, - input_cache_read: 1.75e-7, - output: 0.000014, - output_reasoning_tokens: 0.000014, - output_reasoning: 0.000014, - }, - }, - ], - }, - { - modelName: "gpt-5.2-pro", - matchPattern: "(?i)^(openai/)?(gpt-5.2-pro)$", - startDate: "2025-12-12T09:00:06.513Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000021, - output: 0.000168, - output_reasoning_tokens: 0.000168, - output_reasoning: 0.000168, - }, - }, - ], - }, - { - modelName: "gpt-5.2-pro-2025-12-11", - matchPattern: "(?i)^(openai/)?(gpt-5.2-pro-2025-12-11)$", - startDate: "2025-12-12T09:00:06.513Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.000021, - output: 0.000168, - output_reasoning_tokens: 0.000168, - output_reasoning: 0.000168, - }, - }, - ], - }, - { - modelName: "gpt-5.4", - matchPattern: "(?i)^(openai/)?(gpt-5.4)$", - startDate: "2026-03-05T00:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.0000025, - input_cached_tokens: 2.5e-7, - input_cache_read: 2.5e-7, - output: 0.000015, - output_reasoning_tokens: 0.000015, - output_reasoning: 0.000015, - }, - }, - ], - }, - { - modelName: "gpt-5.4-pro", - matchPattern: "(?i)^(openai/)?(gpt-5.4-pro)$", - startDate: "2026-03-05T00:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00003, - output: 0.00018, - output_reasoning_tokens: 0.00018, - output_reasoning: 0.00018, - }, - }, - ], - }, - { - modelName: "gpt-5.4-2026-03-05", - matchPattern: "(?i)^(openai/)?(gpt-5.4-2026-03-05)$", - startDate: "2026-03-05T00:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.0000025, - input_cached_tokens: 2.5e-7, - input_cache_read: 2.5e-7, - output: 0.000015, - output_reasoning_tokens: 0.000015, - output_reasoning: 0.000015, - }, - }, - ], - }, - { - modelName: "gpt-5.4-pro-2026-03-05", - matchPattern: "(?i)^(openai/)?(gpt-5.4-pro-2026-03-05)$", - startDate: "2026-03-05T00:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 0.00003, - output: 0.00018, - output_reasoning_tokens: 0.00018, - output_reasoning: 0.00018, - }, - }, - ], - }, - { - modelName: "gpt-5.4-mini", - matchPattern: "(?i)^(openai\\/)?(gpt-5.4-mini)$", - startDate: "2026-03-18T00:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 7.5e-7, - input_cached_tokens: 7.5e-8, - input_cache_read: 7.5e-8, - output: 0.0000045, - }, - }, - ], - }, - { - modelName: "gpt-5.4-mini-2026-03-17", - matchPattern: "(?i)^(openai\\/)?(gpt-5.4-mini-2026-03-17)$", - startDate: "2026-03-18T00:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 7.5e-7, - input_cached_tokens: 7.5e-8, - input_cache_read: 7.5e-8, - output: 0.0000045, - }, - }, - ], - }, - { - modelName: "gpt-5.4-nano", - matchPattern: "(?i)^(openai\\/)?(gpt-5.4-nano)$", - startDate: "2026-03-18T00:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 2e-7, - input_cached_tokens: 2e-8, - input_cache_read: 2e-8, - output: 0.00000125, - }, - }, - ], - }, - { - modelName: "gpt-5.4-nano-2026-03-17", - matchPattern: "(?i)^(openai\\/)?(gpt-5.4-nano-2026-03-17)$", - startDate: "2026-03-18T00:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 2e-7, - input_cached_tokens: 2e-8, - input_cache_read: 2e-8, - output: 0.00000125, - }, - }, - ], - }, - { - modelName: "gemini-3-flash-preview", - matchPattern: "(?i)^(google(ai)?/)?(gemini-3-flash-preview)$", - startDate: "2025-12-21T12:01:42.282Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 5e-7, - input_text: 5e-7, - input_modality_1: 5e-7, - prompt_token_count: 5e-7, - promptTokenCount: 5e-7, - input_cached_tokens: 5e-8, - cached_content_token_count: 5e-8, - output: 0.000003, - output_text: 0.000003, - output_modality_1: 0.000003, - candidates_token_count: 0.000003, - candidatesTokenCount: 0.000003, - thoughtsTokenCount: 0.000003, - thoughts_token_count: 0.000003, - output_reasoning: 0.000003, - }, - }, - ], - }, - { - modelName: "gemini-3.1-flash-lite-preview", - matchPattern: "(?i)^(google(ai)?/)?(gemini-3.1-flash-lite-preview)$", - startDate: "2026-03-03T00:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input: 2.5e-7, - input_modality_1: 2.5e-7, - input_text: 2.5e-7, - prompt_token_count: 2.5e-7, - promptTokenCount: 2.5e-7, - input_cached_tokens: 2.5e-8, - cached_content_token_count: 2.5e-8, - output: 0.0000015, - output_text: 0.0000015, - output_modality_1: 0.0000015, - candidates_token_count: 0.0000015, - candidatesTokenCount: 0.0000015, - thoughtsTokenCount: 0.0000015, - thoughts_token_count: 0.0000015, - output_reasoning: 0.0000015, - input_audio_tokens: 5e-7, - }, - }, - ], - }, - { - modelName: "gemini-live-2.5-flash-native-audio", - matchPattern: "(?i)^(google/)?(gemini-live-2.5-flash-native-audio)$", - startDate: "2026-03-16T00:00:00.000Z", - pricingTiers: [ - { - name: "Standard", - isDefault: true, - priority: 0, - conditions: [], - prices: { - input_text: 5e-7, - input_audio: 0.000003, - input_image: 0.000003, - output_text: 0.000002, - output_audio: 0.000012, - }, - }, - ], - }, + "prices": { + "input": 0.000004, + "input_modality_1": 0.000004, + "input_text": 0.000004, + "prompt_token_count": 0.000004, + "promptTokenCount": 0.000004, + "input_cached_tokens": 4e-7, + "cached_content_token_count": 4e-7, + "output": 0.000018, + "output_text": 0.000018, + "output_modality_1": 0.000018, + "candidates_token_count": 0.000018, + "candidatesTokenCount": 0.000018, + "thoughtsTokenCount": 0.000018, + "thoughts_token_count": 0.000018, + "output_reasoning": 0.000018 + } + } + ] + }, + { + "modelName": "gpt-5.2", + "matchPattern": "(?i)^(openai/)?(gpt-5.2)$", + "startDate": "2025-12-12T09:00:06.513Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000175, + "input_cached_tokens": 1.75e-7, + "input_cache_read": 1.75e-7, + "output": 0.000014, + "output_reasoning_tokens": 0.000014, + "output_reasoning": 0.000014 + } + } + ] + }, + { + "modelName": "gpt-5.2-2025-12-11", + "matchPattern": "(?i)^(openai/)?(gpt-5.2-2025-12-11)$", + "startDate": "2025-12-12T09:00:06.513Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000175, + "input_cached_tokens": 1.75e-7, + "input_cache_read": 1.75e-7, + "output": 0.000014, + "output_reasoning_tokens": 0.000014, + "output_reasoning": 0.000014 + } + } + ] + }, + { + "modelName": "gpt-5.2-pro", + "matchPattern": "(?i)^(openai/)?(gpt-5.2-pro)$", + "startDate": "2025-12-12T09:00:06.513Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000021, + "output": 0.000168, + "output_reasoning_tokens": 0.000168, + "output_reasoning": 0.000168 + } + } + ] + }, + { + "modelName": "gpt-5.2-pro-2025-12-11", + "matchPattern": "(?i)^(openai/)?(gpt-5.2-pro-2025-12-11)$", + "startDate": "2025-12-12T09:00:06.513Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000021, + "output": 0.000168, + "output_reasoning_tokens": 0.000168, + "output_reasoning": 0.000168 + } + } + ] + }, + { + "modelName": "gpt-5.4", + "matchPattern": "(?i)^(openai/)?(gpt-5.4)$", + "startDate": "2026-03-05T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 2.5e-7, + "input_cache_read": 2.5e-7, + "output": 0.000015, + "output_reasoning_tokens": 0.000015, + "output_reasoning": 0.000015 + } + } + ] + }, + { + "modelName": "gpt-5.4-pro", + "matchPattern": "(?i)^(openai/)?(gpt-5.4-pro)$", + "startDate": "2026-03-05T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00018, + "output_reasoning_tokens": 0.00018, + "output_reasoning": 0.00018 + } + } + ] + }, + { + "modelName": "gpt-5.4-2026-03-05", + "matchPattern": "(?i)^(openai/)?(gpt-5.4-2026-03-05)$", + "startDate": "2026-03-05T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 2.5e-7, + "input_cache_read": 2.5e-7, + "output": 0.000015, + "output_reasoning_tokens": 0.000015, + "output_reasoning": 0.000015 + } + } + ] + }, + { + "modelName": "gpt-5.4-pro-2026-03-05", + "matchPattern": "(?i)^(openai/)?(gpt-5.4-pro-2026-03-05)$", + "startDate": "2026-03-05T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00018, + "output_reasoning_tokens": 0.00018, + "output_reasoning": 0.00018 + } + } + ] + }, + { + "modelName": "gpt-5.4-mini", + "matchPattern": "(?i)^(openai\\/)?(gpt-5.4-mini)$", + "startDate": "2026-03-18T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 7.5e-7, + "input_cached_tokens": 7.5e-8, + "input_cache_read": 7.5e-8, + "output": 0.0000045 + } + } + ] + }, + { + "modelName": "gpt-5.4-mini-2026-03-17", + "matchPattern": "(?i)^(openai\\/)?(gpt-5.4-mini-2026-03-17)$", + "startDate": "2026-03-18T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 7.5e-7, + "input_cached_tokens": 7.5e-8, + "input_cache_read": 7.5e-8, + "output": 0.0000045 + } + } + ] + }, + { + "modelName": "gpt-5.4-nano", + "matchPattern": "(?i)^(openai\\/)?(gpt-5.4-nano)$", + "startDate": "2026-03-18T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2e-7, + "input_cached_tokens": 2e-8, + "input_cache_read": 2e-8, + "output": 0.00000125 + } + } + ] + }, + { + "modelName": "gpt-5.4-nano-2026-03-17", + "matchPattern": "(?i)^(openai\\/)?(gpt-5.4-nano-2026-03-17)$", + "startDate": "2026-03-18T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2e-7, + "input_cached_tokens": 2e-8, + "input_cache_read": 2e-8, + "output": 0.00000125 + } + } + ] + }, + { + "modelName": "gemini-3-flash-preview", + "matchPattern": "(?i)^(google(ai)?/)?(gemini-3-flash-preview)$", + "startDate": "2025-12-21T12:01:42.282Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "input_text": 5e-7, + "input_modality_1": 5e-7, + "prompt_token_count": 5e-7, + "promptTokenCount": 5e-7, + "input_cached_tokens": 5e-8, + "cached_content_token_count": 5e-8, + "output": 0.000003, + "output_text": 0.000003, + "output_modality_1": 0.000003, + "candidates_token_count": 0.000003, + "candidatesTokenCount": 0.000003, + "thoughtsTokenCount": 0.000003, + "thoughts_token_count": 0.000003, + "output_reasoning": 0.000003 + } + } + ] + }, + { + "modelName": "gemini-3.1-flash-lite-preview", + "matchPattern": "(?i)^(google(ai)?/)?(gemini-3.1-flash-lite-preview)$", + "startDate": "2026-03-03T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "input_modality_1": 2.5e-7, + "input_text": 2.5e-7, + "prompt_token_count": 2.5e-7, + "promptTokenCount": 2.5e-7, + "input_cached_tokens": 2.5e-8, + "cached_content_token_count": 2.5e-8, + "output": 0.0000015, + "output_text": 0.0000015, + "output_modality_1": 0.0000015, + "candidates_token_count": 0.0000015, + "candidatesTokenCount": 0.0000015, + "thoughtsTokenCount": 0.0000015, + "thoughts_token_count": 0.0000015, + "output_reasoning": 0.0000015, + "input_audio_tokens": 5e-7 + } + } + ] + }, + { + "modelName": "gemini-live-2.5-flash-native-audio", + "matchPattern": "(?i)^(google/)?(gemini-live-2.5-flash-native-audio)$", + "startDate": "2026-03-16T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text": 5e-7, + "input_audio": 0.000003, + "input_image": 0.000003, + "output_text": 0.000002, + "output_audio": 0.000012 + } + } + ] + } ]; diff --git a/internal-packages/llm-model-catalog/src/modelCatalog.ts b/internal-packages/llm-model-catalog/src/modelCatalog.ts index d90eb1a992b..71ae921c3e7 100644 --- a/internal-packages/llm-model-catalog/src/modelCatalog.ts +++ b/internal-packages/llm-model-catalog/src/modelCatalog.ts @@ -5,564 +5,677 @@ import type { ModelCatalogEntry } from "./types.js"; export const modelCatalog: Record<string, ModelCatalogEntry> = { "chatgpt-4o-latest": { - provider: "openai", - description: - "OpenAI's flagship multimodal model optimized for speed and cost, capable of processing text, images, and audio with strong performance across reasoning, coding, and creative tasks.", - contextWindow: 128000, - maxOutputTokens: 16384, - capabilities: [ + "provider": "openai", + "description": "OpenAI's flagship multimodal model optimized for speed and cost, capable of processing text, images, and audio with strong performance across reasoning, coding, and creative tasks.", + "contextWindow": 128000, + "maxOutputTokens": 16384, + "capabilities": [ "vision", "tool_use", "streaming", "json_mode", "audio_input", "audio_output", - "fine_tunable", + "fine_tunable" ], - releaseDate: "2024-05-13", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T10:55:46.469Z", - baseModelName: "chatgpt-4o", + "releaseDate": "2024-05-13", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T10:55:46.469Z", + "baseModelName": "chatgpt-4o" }, "claude-1.1": { - provider: "anthropic", - description: - "An early-generation Claude model from Anthropic, offering basic conversational and text completion capabilities. It was quickly superseded by Claude 1.2, 1.3, and the Claude 2 family.", - contextWindow: 9000, - maxOutputTokens: 8191, - capabilities: ["streaming"], - releaseDate: "2023-03-14", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: null, - knowledgeCutoff: null, - resolvedAt: "2026-03-24T10:55:47.906Z", - baseModelName: null, + "provider": "anthropic", + "description": "An early-generation Claude model from Anthropic, offering basic conversational and text completion capabilities. It was quickly superseded by Claude 1.2, 1.3, and the Claude 2 family.", + "contextWindow": 9000, + "maxOutputTokens": 8191, + "capabilities": [ + "streaming" + ], + "releaseDate": "2023-03-14", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": null, + "knowledgeCutoff": null, + "resolvedAt": "2026-03-24T10:55:47.906Z", + "baseModelName": null }, "claude-1.2": { - provider: "anthropic", - description: - "An early-generation Anthropic model, part of the original Claude 1.x family. It offered improved performance over Claude 1.0 but was quickly superseded by Claude 1.3 and later model families.", - contextWindow: 9000, - maxOutputTokens: 8191, - capabilities: ["streaming"], - releaseDate: null, - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: null, - knowledgeCutoff: null, - resolvedAt: "2026-03-24T10:55:46.760Z", - baseModelName: null, + "provider": "anthropic", + "description": "An early-generation Anthropic model, part of the original Claude 1.x family. It offered improved performance over Claude 1.0 but was quickly superseded by Claude 1.3 and later model families.", + "contextWindow": 9000, + "maxOutputTokens": 8191, + "capabilities": [ + "streaming" + ], + "releaseDate": null, + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": null, + "knowledgeCutoff": null, + "resolvedAt": "2026-03-24T10:55:46.760Z", + "baseModelName": null }, "claude-1.3": { - provider: "anthropic", - description: - "Early-generation Claude model from Anthropic, offering improved performance over Claude 1.0-1.2 in reasoning and instruction-following tasks.", - contextWindow: 100000, - maxOutputTokens: null, - capabilities: ["streaming"], - releaseDate: "2023-03-14", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: null, - knowledgeCutoff: null, - resolvedAt: "2026-03-24T10:55:46.227Z", - baseModelName: null, + "provider": "anthropic", + "description": "Early-generation Claude model from Anthropic, offering improved performance over Claude 1.0-1.2 in reasoning and instruction-following tasks.", + "contextWindow": 100000, + "maxOutputTokens": null, + "capabilities": [ + "streaming" + ], + "releaseDate": "2023-03-14", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": null, + "knowledgeCutoff": null, + "resolvedAt": "2026-03-24T10:55:46.227Z", + "baseModelName": null }, "claude-2.0": { - provider: "anthropic", - description: - "Anthropic's second-generation large language model, offering improved performance over Claude 1.x with longer context support. Succeeded by Claude 2.1 and later the Claude 3 family.", - contextWindow: 100000, - maxOutputTokens: 4096, - capabilities: ["streaming"], - releaseDate: "2023-07-11", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: null, - knowledgeCutoff: "2023-02-01", - resolvedAt: "2026-03-24T10:55:45.922Z", - baseModelName: null, + "provider": "anthropic", + "description": "Anthropic's second-generation large language model, offering improved performance over Claude 1.x with longer context support. Succeeded by Claude 2.1 and later the Claude 3 family.", + "contextWindow": 100000, + "maxOutputTokens": 4096, + "capabilities": [ + "streaming" + ], + "releaseDate": "2023-07-11", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": null, + "knowledgeCutoff": "2023-02-01", + "resolvedAt": "2026-03-24T10:55:45.922Z", + "baseModelName": null }, "claude-2.1": { - provider: "anthropic", - description: - "Anthropic's Claude 2.1 model featuring a 200K context window, reduced hallucination rates compared to Claude 2.0, and improved accuracy on long document comprehension.", - contextWindow: 200000, - maxOutputTokens: 4096, - capabilities: ["streaming", "tool_use"], - releaseDate: "2023-11-21", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: null, - knowledgeCutoff: "2023-01-01", - resolvedAt: "2026-03-24T10:56:22.743Z", - baseModelName: null, + "provider": "anthropic", + "description": "Anthropic's Claude 2.1 model featuring a 200K context window, reduced hallucination rates compared to Claude 2.0, and improved accuracy on long document comprehension.", + "contextWindow": 200000, + "maxOutputTokens": 4096, + "capabilities": [ + "streaming", + "tool_use" + ], + "releaseDate": "2023-11-21", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": null, + "knowledgeCutoff": "2023-01-01", + "resolvedAt": "2026-03-24T10:56:22.743Z", + "baseModelName": null }, "claude-3-5-haiku-20241022": { - provider: "anthropic", - description: - "Anthropic's fastest and most cost-effective model in the Claude 3.5 family, optimized for speed and efficiency while maintaining strong performance across common tasks.", - contextWindow: 200000, - maxOutputTokens: 8192, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2024-10-22", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-07-01", - resolvedAt: "2026-03-24T10:56:25.724Z", - baseModelName: "claude-3-5-haiku", + "provider": "anthropic", + "description": "Anthropic's fastest and most cost-effective model in the Claude 3.5 family, optimized for speed and efficiency while maintaining strong performance across common tasks.", + "contextWindow": 200000, + "maxOutputTokens": 8192, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2024-10-22", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-07-01", + "resolvedAt": "2026-03-24T10:56:25.724Z", + "baseModelName": "claude-3-5-haiku" }, "claude-3-5-sonnet-20240620": { - provider: "anthropic", - description: - "Anthropic's Claude 3.5 Sonnet is a mid-tier model balancing intelligence and speed, excelling at coding, analysis, and vision tasks while being faster and cheaper than Opus.", - contextWindow: 200000, - maxOutputTokens: 8192, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2024-06-20", - isHidden: true, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-04-01", - resolvedAt: "2026-03-24T10:56:35.401Z", - baseModelName: "claude-3-5-sonnet", + "provider": "anthropic", + "description": "Anthropic's Claude 3.5 Sonnet is a mid-tier model balancing intelligence and speed, excelling at coding, analysis, and vision tasks while being faster and cheaper than Opus.", + "contextWindow": 200000, + "maxOutputTokens": 8192, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2024-06-20", + "isHidden": true, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-04-01", + "resolvedAt": "2026-03-24T10:56:35.401Z", + "baseModelName": "claude-3-5-sonnet" }, "claude-3-haiku-20240307": { - provider: "anthropic", - description: - "Anthropic's fastest and most compact Claude 3 model, optimized for speed and cost-efficiency while maintaining strong performance on everyday tasks.", - contextWindow: 200000, - maxOutputTokens: 4096, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2024-03-13", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-08-01", - resolvedAt: "2026-03-24T10:56:25.288Z", - baseModelName: "claude-3-haiku", + "provider": "anthropic", + "description": "Anthropic's fastest and most compact Claude 3 model, optimized for speed and cost-efficiency while maintaining strong performance on everyday tasks.", + "contextWindow": 200000, + "maxOutputTokens": 4096, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2024-03-13", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-08-01", + "resolvedAt": "2026-03-24T10:56:25.288Z", + "baseModelName": "claude-3-haiku" }, "claude-3-opus-20240229": { - provider: "anthropic", - description: - "Anthropic's most capable model in the Claude 3 family, excelling at complex analysis, nuanced content generation, and advanced reasoning tasks.", - contextWindow: 200000, - maxOutputTokens: 4096, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2024-03-04", - isHidden: true, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2023-08-01", - resolvedAt: "2026-03-24T10:56:26.008Z", - baseModelName: "claude-3-opus", + "provider": "anthropic", + "description": "Anthropic's most capable model in the Claude 3 family, excelling at complex analysis, nuanced content generation, and advanced reasoning tasks.", + "contextWindow": 200000, + "maxOutputTokens": 4096, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2024-03-04", + "isHidden": true, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2023-08-01", + "resolvedAt": "2026-03-24T10:56:26.008Z", + "baseModelName": "claude-3-opus" }, "claude-3-sonnet-20240229": { - provider: "anthropic", - description: - "Mid-tier model in Anthropic's Claude 3 family, balancing performance and speed for a wide range of tasks including analysis, coding, and content generation.", - contextWindow: 200000, - maxOutputTokens: 4096, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2024-03-04", - isHidden: true, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-02-01", - resolvedAt: "2026-03-24T10:56:59.532Z", - baseModelName: "claude-3-sonnet", + "provider": "anthropic", + "description": "Mid-tier model in Anthropic's Claude 3 family, balancing performance and speed for a wide range of tasks including analysis, coding, and content generation.", + "contextWindow": 200000, + "maxOutputTokens": 4096, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2024-03-04", + "isHidden": true, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-02-01", + "resolvedAt": "2026-03-24T10:56:59.532Z", + "baseModelName": "claude-3-sonnet" }, "claude-3.5-haiku-latest": { - provider: "anthropic", - description: - "Anthropic's fastest and most cost-effective model in the Claude 3.5 family, optimized for speed and efficiency while maintaining strong performance across a wide range of tasks.", - contextWindow: 200000, - maxOutputTokens: 8192, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2024-10-29", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-07-01", - resolvedAt: "2026-03-24T10:57:04.392Z", - baseModelName: "claude-3.5-haiku", + "provider": "anthropic", + "description": "Anthropic's fastest and most cost-effective model in the Claude 3.5 family, optimized for speed and efficiency while maintaining strong performance across a wide range of tasks.", + "contextWindow": 200000, + "maxOutputTokens": 8192, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2024-10-29", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-07-01", + "resolvedAt": "2026-03-24T10:57:04.392Z", + "baseModelName": "claude-3.5-haiku" }, "claude-3.5-sonnet-20241022": { - provider: "anthropic", - description: - "Anthropic's mid-tier model offering strong reasoning, coding, and analysis capabilities at a balance of speed and intelligence, positioned between Haiku and Opus in the Claude 3.5 family.", - contextWindow: 200000, - maxOutputTokens: 8192, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2024-06-20", - isHidden: true, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-04-01", - resolvedAt: "2026-03-24T10:57:13.346Z", - baseModelName: "claude-3.5-sonnet", + "provider": "anthropic", + "description": "Anthropic's mid-tier model offering strong reasoning, coding, and analysis capabilities at a balance of speed and intelligence, positioned between Haiku and Opus in the Claude 3.5 family.", + "contextWindow": 200000, + "maxOutputTokens": 8192, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2024-06-20", + "isHidden": true, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-04-01", + "resolvedAt": "2026-03-24T10:57:13.346Z", + "baseModelName": "claude-3.5-sonnet" }, "claude-3.5-sonnet-latest": { - provider: "anthropic", - description: - "Anthropic's mid-tier model offering strong reasoning, coding, and analysis capabilities at a balance of speed and intelligence, positioned between Haiku and Opus in the Claude 3.5 family.", - contextWindow: 200000, - maxOutputTokens: 8192, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2024-06-20", - isHidden: true, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-04-01", - resolvedAt: "2026-03-24T10:57:13.346Z", - baseModelName: "claude-3.5-sonnet", + "provider": "anthropic", + "description": "Anthropic's mid-tier model offering strong reasoning, coding, and analysis capabilities at a balance of speed and intelligence, positioned between Haiku and Opus in the Claude 3.5 family.", + "contextWindow": 200000, + "maxOutputTokens": 8192, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2024-06-20", + "isHidden": true, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-04-01", + "resolvedAt": "2026-03-24T10:57:13.346Z", + "baseModelName": "claude-3.5-sonnet" }, "claude-3.7-sonnet-20250219": { - provider: "anthropic", - description: - "Anthropic's Claude 3.7 Sonnet is a hybrid reasoning model that introduced extended thinking capabilities, offering strong performance on coding, math, and complex reasoning tasks.", - contextWindow: 200000, - maxOutputTokens: 16384, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-02-24", - isHidden: true, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-04-01", - resolvedAt: "2026-03-24T10:57:12.967Z", - baseModelName: "claude-3.7-sonnet", + "provider": "anthropic", + "description": "Anthropic's Claude 3.7 Sonnet is a hybrid reasoning model that introduced extended thinking capabilities, offering strong performance on coding, math, and complex reasoning tasks.", + "contextWindow": 200000, + "maxOutputTokens": 16384, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-02-24", + "isHidden": true, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-04-01", + "resolvedAt": "2026-03-24T10:57:12.967Z", + "baseModelName": "claude-3.7-sonnet" }, "claude-3.7-sonnet-latest": { - provider: "anthropic", - description: - "Anthropic's Claude 3.7 Sonnet is a hybrid reasoning model that introduced extended thinking capabilities, offering strong performance on coding, math, and complex reasoning tasks.", - contextWindow: 200000, - maxOutputTokens: 16384, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-02-24", - isHidden: true, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-04-01", - resolvedAt: "2026-03-24T10:57:12.967Z", - baseModelName: "claude-3.7-sonnet", + "provider": "anthropic", + "description": "Anthropic's Claude 3.7 Sonnet is a hybrid reasoning model that introduced extended thinking capabilities, offering strong performance on coding, math, and complex reasoning tasks.", + "contextWindow": 200000, + "maxOutputTokens": 16384, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-02-24", + "isHidden": true, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-04-01", + "resolvedAt": "2026-03-24T10:57:12.967Z", + "baseModelName": "claude-3.7-sonnet" }, "claude-haiku-4-5-20251001": { - provider: "anthropic", - description: - "Anthropic's fastest model with near-frontier intelligence, optimized for speed and cost efficiency while supporting extended thinking and vision.", - contextWindow: 200000, - maxOutputTokens: 64000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-10-01", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-07-01", - resolvedAt: "2026-03-24T10:57:29.685Z", - baseModelName: "claude-haiku-4-5", + "provider": "anthropic", + "description": "Anthropic's fastest model with near-frontier intelligence, optimized for speed and cost efficiency while supporting extended thinking and vision.", + "contextWindow": 200000, + "maxOutputTokens": 64000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-10-01", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-07-01", + "resolvedAt": "2026-03-24T10:57:29.685Z", + "baseModelName": "claude-haiku-4-5" }, "claude-instant-1": { - provider: "anthropic", - description: - "Anthropic's fast and cost-effective model optimized for speed and efficiency, positioned as a lighter alternative to Claude 1.x for tasks requiring lower latency.", - contextWindow: 100000, - maxOutputTokens: 8191, - capabilities: ["streaming"], - releaseDate: "2023-03-14", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: "2025-01-06", - knowledgeCutoff: "2023-01-01", - resolvedAt: "2026-03-24T10:57:36.888Z", - baseModelName: null, + "provider": "anthropic", + "description": "Anthropic's fast and cost-effective model optimized for speed and efficiency, positioned as a lighter alternative to Claude 1.x for tasks requiring lower latency.", + "contextWindow": 100000, + "maxOutputTokens": 8191, + "capabilities": [ + "streaming" + ], + "releaseDate": "2023-03-14", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": "2025-01-06", + "knowledgeCutoff": "2023-01-01", + "resolvedAt": "2026-03-24T10:57:36.888Z", + "baseModelName": null }, "claude-instant-1.2": { - provider: "anthropic", - description: - "Anthropic's fast and cost-effective model, optimized for speed and efficiency while maintaining strong performance on conversational and text generation tasks.", - contextWindow: 100000, - maxOutputTokens: 8191, - capabilities: ["streaming"], - releaseDate: "2023-08-09", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: null, - knowledgeCutoff: "2023-01-01", - resolvedAt: "2026-03-24T10:57:41.865Z", - baseModelName: null, + "provider": "anthropic", + "description": "Anthropic's fast and cost-effective model, optimized for speed and efficiency while maintaining strong performance on conversational and text generation tasks.", + "contextWindow": 100000, + "maxOutputTokens": 8191, + "capabilities": [ + "streaming" + ], + "releaseDate": "2023-08-09", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": null, + "knowledgeCutoff": "2023-01-01", + "resolvedAt": "2026-03-24T10:57:41.865Z", + "baseModelName": null }, "claude-opus-4-1-20250805": { - provider: "anthropic", - description: - "Anthropic's hybrid reasoning model with strong software engineering and agentic capabilities, scoring 74.5% on SWE-bench Verified. Supports both rapid responses and step-by-step extended thinking.", - contextWindow: 200000, - maxOutputTokens: 32000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-08-05", - isHidden: true, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-03-01", - resolvedAt: "2026-03-24T10:58:36.876Z", - baseModelName: "claude-opus-4-1", + "provider": "anthropic", + "description": "Anthropic's hybrid reasoning model with strong software engineering and agentic capabilities, scoring 74.5% on SWE-bench Verified. Supports both rapid responses and step-by-step extended thinking.", + "contextWindow": 200000, + "maxOutputTokens": 32000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-08-05", + "isHidden": true, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-03-01", + "resolvedAt": "2026-03-24T10:58:36.876Z", + "baseModelName": "claude-opus-4-1" }, "claude-opus-4-20250514": { - provider: "anthropic", - description: - "Anthropic's flagship model from the Claude 4 family, excelling at complex coding tasks, long-running agent workflows, and deep reasoning with extended thinking support.", - contextWindow: 200000, - maxOutputTokens: 32000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-05-14", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-03-01", - resolvedAt: "2026-03-24T10:58:47.518Z", - baseModelName: "claude-opus-4", + "provider": "anthropic", + "description": "Anthropic's flagship model from the Claude 4 family, excelling at complex coding tasks, long-running agent workflows, and deep reasoning with extended thinking support.", + "contextWindow": 200000, + "maxOutputTokens": 32000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-05-14", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-03-01", + "resolvedAt": "2026-03-24T10:58:47.518Z", + "baseModelName": "claude-opus-4" }, "claude-opus-4-5-20251101": { - provider: "anthropic", - description: - "Anthropic's flagship intelligence model released in November 2025, excelling at complex reasoning, vision, and extended thinking with the best performance in Anthropic's lineup before Opus 4.6.", - contextWindow: 200000, - maxOutputTokens: 64000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-11-01", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-03-01", - resolvedAt: "2026-03-24T10:58:48.961Z", - baseModelName: "claude-opus-4-5", + "provider": "anthropic", + "description": "Anthropic's flagship intelligence model released in November 2025, excelling at complex reasoning, vision, and extended thinking with the best performance in Anthropic's lineup before Opus 4.6.", + "contextWindow": 200000, + "maxOutputTokens": 64000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-11-01", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-03-01", + "resolvedAt": "2026-03-24T10:58:48.961Z", + "baseModelName": "claude-opus-4-5" }, "claude-opus-4-6": { - provider: "anthropic", - description: - "Anthropic's most intelligent model, optimized for building agents and coding with exceptional reasoning capabilities and extended agentic task horizons.", - contextWindow: 1000000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2026-02-05", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-05-01", - resolvedAt: "2026-03-24T10:58:42.061Z", - baseModelName: null, + "provider": "anthropic", + "description": "Anthropic's most intelligent model, optimized for building agents and coding with exceptional reasoning capabilities and extended agentic task horizons.", + "contextWindow": 1000000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2026-02-05", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-05-01", + "resolvedAt": "2026-03-24T10:58:42.061Z", + "baseModelName": null }, "claude-sonnet-4-20250514": { - provider: "anthropic", - description: - "Anthropic's balanced Claude 4 model offering strong coding, reasoning, and multilingual performance at moderate cost. Now a legacy model superseded by Claude Sonnet 4.5 and 4.6.", - contextWindow: 200000, - maxOutputTokens: 64000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-05-14", - isHidden: true, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-03-01", - resolvedAt: "2026-03-24T10:58:39.601Z", - baseModelName: "claude-sonnet-4", + "provider": "anthropic", + "description": "Anthropic's balanced Claude 4 model offering strong coding, reasoning, and multilingual performance at moderate cost. Now a legacy model superseded by Claude Sonnet 4.5 and 4.6.", + "contextWindow": 200000, + "maxOutputTokens": 64000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-05-14", + "isHidden": true, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-03-01", + "resolvedAt": "2026-03-24T10:58:39.601Z", + "baseModelName": "claude-sonnet-4" }, "claude-sonnet-4-5-20250929": { - provider: "anthropic", - description: - "Anthropic's high-performance mid-tier model with strong coding, reasoning, and multi-step problem solving capabilities. Successor to Claude Sonnet 4, offering improved benchmarks at the same price point.", - contextWindow: 200000, - maxOutputTokens: 64000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-09-29", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-01-01", - resolvedAt: "2026-03-24T10:59:54.426Z", - baseModelName: "claude-sonnet-4-5", + "provider": "anthropic", + "description": "Anthropic's high-performance mid-tier model with strong coding, reasoning, and multi-step problem solving capabilities. Successor to Claude Sonnet 4, offering improved benchmarks at the same price point.", + "contextWindow": 200000, + "maxOutputTokens": 64000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-09-29", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-01-01", + "resolvedAt": "2026-03-24T10:59:54.426Z", + "baseModelName": "claude-sonnet-4-5" }, "claude-sonnet-4-6": { - provider: "anthropic", - description: - "Anthropic's best combination of speed and intelligence, excelling at coding, agentic tasks, and computer use, with a 1M token context window and performance rivaling prior Opus-class models.", - contextWindow: 1000000, - maxOutputTokens: 64000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2026-02-17", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2026-01-01", - resolvedAt: "2026-03-24T10:59:59.014Z", - baseModelName: null, + "provider": "anthropic", + "description": "Anthropic's best combination of speed and intelligence, excelling at coding, agentic tasks, and computer use, with a 1M token context window and performance rivaling prior Opus-class models.", + "contextWindow": 1000000, + "maxOutputTokens": 64000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2026-02-17", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2026-01-01", + "resolvedAt": "2026-03-24T10:59:59.014Z", + "baseModelName": null }, "claude-sonnet-4-latest": { - provider: "anthropic", - description: - "Anthropic's balanced Claude 4 model offering strong coding, reasoning, and multilingual performance at moderate cost. Now a legacy model superseded by Claude Sonnet 4.5 and 4.6.", - contextWindow: 200000, - maxOutputTokens: 64000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-05-14", - isHidden: true, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-03-01", - resolvedAt: "2026-03-24T10:58:39.601Z", - baseModelName: "claude-sonnet-4", + "provider": "anthropic", + "description": "Anthropic's balanced Claude 4 model offering strong coding, reasoning, and multilingual performance at moderate cost. Now a legacy model superseded by Claude Sonnet 4.5 and 4.6.", + "contextWindow": 200000, + "maxOutputTokens": 64000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-05-14", + "isHidden": true, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-03-01", + "resolvedAt": "2026-03-24T10:58:39.601Z", + "baseModelName": "claude-sonnet-4" }, "gemini-1.0-pro": { - provider: "google", - description: - "Google's first-generation Gemini Pro model, a mid-size multimodal model designed for text generation, reasoning, and chat applications. Succeeded by Gemini 1.5 Pro.", - contextWindow: 32760, - maxOutputTokens: 8192, - capabilities: ["tool_use", "streaming", "json_mode"], - releaseDate: "2023-12-13", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: "2025-02-15", - knowledgeCutoff: "2023-04-01", - resolvedAt: "2026-03-24T10:59:26.767Z", - baseModelName: null, + "provider": "google", + "description": "Google's first-generation Gemini Pro model, a mid-size multimodal model designed for text generation, reasoning, and chat applications. Succeeded by Gemini 1.5 Pro.", + "contextWindow": 32760, + "maxOutputTokens": 8192, + "capabilities": [ + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2023-12-13", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": "2025-02-15", + "knowledgeCutoff": "2023-04-01", + "resolvedAt": "2026-03-24T10:59:26.767Z", + "baseModelName": null }, "gemini-1.0-pro-001": { - provider: "google", - description: - "Google's first-generation Pro model optimized for text generation, reasoning, and multi-turn conversation tasks, part of the original Gemini 1.0 lineup.", - contextWindow: 30720, - maxOutputTokens: 2048, - capabilities: ["tool_use", "streaming", "json_mode"], - releaseDate: "2024-02-15", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: "2025-02-15", - knowledgeCutoff: "2023-04-01", - resolvedAt: "2026-03-24T10:59:27.391Z", - baseModelName: null, + "provider": "google", + "description": "Google's first-generation Pro model optimized for text generation, reasoning, and multi-turn conversation tasks, part of the original Gemini 1.0 lineup.", + "contextWindow": 30720, + "maxOutputTokens": 2048, + "capabilities": [ + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2024-02-15", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": "2025-02-15", + "knowledgeCutoff": "2023-04-01", + "resolvedAt": "2026-03-24T10:59:27.391Z", + "baseModelName": null }, "gemini-1.0-pro-latest": { - provider: "google", - description: - "Google's first-generation Gemini Pro model, a mid-size multimodal model designed for text generation, reasoning, and chat applications. Succeeded by Gemini 1.5 Pro.", - contextWindow: 32760, - maxOutputTokens: 8192, - capabilities: ["tool_use", "streaming", "json_mode"], - releaseDate: "2023-12-13", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: "2025-02-15", - knowledgeCutoff: "2023-04-01", - resolvedAt: "2026-03-24T10:59:26.767Z", - baseModelName: "gemini-1.0-pro", + "provider": "google", + "description": "Google's first-generation Gemini Pro model, a mid-size multimodal model designed for text generation, reasoning, and chat applications. Succeeded by Gemini 1.5 Pro.", + "contextWindow": 32760, + "maxOutputTokens": 8192, + "capabilities": [ + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2023-12-13", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": "2025-02-15", + "knowledgeCutoff": "2023-04-01", + "resolvedAt": "2026-03-24T10:59:26.767Z", + "baseModelName": "gemini-1.0-pro" }, "gemini-1.5-pro-latest": { - provider: "google", - description: - "Google's mid-size multimodal model with a massive context window, strong at long-document understanding, code generation, and multi-turn conversation.", - contextWindow: 2097152, - maxOutputTokens: 8192, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "audio_input"], - releaseDate: "2024-02-15", - isHidden: true, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2025-09-24", - knowledgeCutoff: "2024-04-01", - resolvedAt: "2026-03-24T10:59:25.463Z", - baseModelName: "gemini-1.5-pro", + "provider": "google", + "description": "Google's mid-size multimodal model with a massive context window, strong at long-document understanding, code generation, and multi-turn conversation.", + "contextWindow": 2097152, + "maxOutputTokens": 8192, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "audio_input" + ], + "releaseDate": "2024-02-15", + "isHidden": true, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2025-09-24", + "knowledgeCutoff": "2024-04-01", + "resolvedAt": "2026-03-24T10:59:25.463Z", + "baseModelName": "gemini-1.5-pro" }, "gemini-2.0-flash": { - provider: "google", - description: - "Google's second-generation workhorse model optimized for speed, with native tool use, multimodal input (text, images, audio, video), and a 1M token context window.", - contextWindow: 1048576, - maxOutputTokens: 8192, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "code_execution", "audio_input"], - releaseDate: "2025-02-05", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2026-06-01", - knowledgeCutoff: "2024-06-01", - resolvedAt: "2026-03-24T11:01:15.429Z", - baseModelName: null, + "provider": "google", + "description": "Google's second-generation workhorse model optimized for speed, with native tool use, multimodal input (text, images, audio, video), and a 1M token context window.", + "contextWindow": 1048576, + "maxOutputTokens": 8192, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "code_execution", + "audio_input" + ], + "releaseDate": "2025-02-05", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2026-06-01", + "knowledgeCutoff": "2024-06-01", + "resolvedAt": "2026-03-24T11:01:15.429Z", + "baseModelName": null }, "gemini-2.0-flash-001": { - provider: "google", - description: - "Google's fast and efficient multimodal model that outperforms Gemini 1.5 Pro on key benchmarks at twice the speed, supporting text, image, audio, and video inputs with native tool use.", - contextWindow: 1048576, - maxOutputTokens: 8192, - capabilities: [ + "provider": "google", + "description": "Google's fast and efficient multimodal model that outperforms Gemini 1.5 Pro on key benchmarks at twice the speed, supporting text, image, audio, and video inputs with native tool use.", + "contextWindow": 1048576, + "maxOutputTokens": 8192, + "capabilities": [ "vision", "tool_use", "streaming", @@ -570,1614 +683,1890 @@ export const modelCatalog: Record<string, ModelCatalogEntry> = { "audio_input", "image_generation", "audio_output", - "code_execution", + "code_execution" ], - releaseDate: "2025-02-05", - isHidden: true, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2026-06-01", - knowledgeCutoff: "2024-08-01", - resolvedAt: "2026-03-24T11:01:04.084Z", - baseModelName: null, + "releaseDate": "2025-02-05", + "isHidden": true, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2026-06-01", + "knowledgeCutoff": "2024-08-01", + "resolvedAt": "2026-03-24T11:01:04.084Z", + "baseModelName": null }, "gemini-2.0-flash-lite-preview": { - provider: "google", - description: - "A lightweight, cost-efficient variant of Gemini 2.0 Flash optimized for low latency and high throughput, supporting multimodal input with text output.", - contextWindow: 1048576, - maxOutputTokens: 8192, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2025-02-05", - isHidden: true, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2026-06-01", - knowledgeCutoff: "2024-06-01", - resolvedAt: "2026-03-24T11:00:56.775Z", - baseModelName: null, + "provider": "google", + "description": "A lightweight, cost-efficient variant of Gemini 2.0 Flash optimized for low latency and high throughput, supporting multimodal input with text output.", + "contextWindow": 1048576, + "maxOutputTokens": 8192, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2025-02-05", + "isHidden": true, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2026-06-01", + "knowledgeCutoff": "2024-06-01", + "resolvedAt": "2026-03-24T11:00:56.775Z", + "baseModelName": null }, "gemini-2.0-flash-lite-preview-02-05": { - provider: "google", - description: - "Google's cost-optimized, low-latency model in the Gemini 2.0 family, designed for high-volume tasks like summarization, multimodal processing, and categorization.", - contextWindow: 1048576, - maxOutputTokens: 8192, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2025-02-05", - isHidden: true, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2025-12-09", - knowledgeCutoff: "2024-06-01", - resolvedAt: "2026-03-24T11:01:34.165Z", - baseModelName: null, + "provider": "google", + "description": "Google's cost-optimized, low-latency model in the Gemini 2.0 family, designed for high-volume tasks like summarization, multimodal processing, and categorization.", + "contextWindow": 1048576, + "maxOutputTokens": 8192, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2025-02-05", + "isHidden": true, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2025-12-09", + "knowledgeCutoff": "2024-06-01", + "resolvedAt": "2026-03-24T11:01:34.165Z", + "baseModelName": null }, "gemini-2.5-flash": { - provider: "google", - description: - "Google's best price-performance model optimized for low-latency, high-volume tasks requiring reasoning, with built-in thinking capabilities and multimodal input support.", - contextWindow: 1048576, - maxOutputTokens: 65536, - capabilities: [ + "provider": "google", + "description": "Google's best price-performance model optimized for low-latency, high-volume tasks requiring reasoning, with built-in thinking capabilities and multimodal input support.", + "contextWindow": 1048576, + "maxOutputTokens": 65536, + "capabilities": [ "vision", "tool_use", "streaming", "json_mode", "extended_thinking", "code_execution", - "audio_input", + "audio_input" ], - releaseDate: "2025-06-01", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: false, - deprecationDate: null, - knowledgeCutoff: "2025-01-01", - resolvedAt: "2026-03-24T11:01:25.200Z", - baseModelName: null, + "releaseDate": "2025-06-01", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": false, + "deprecationDate": null, + "knowledgeCutoff": "2025-01-01", + "resolvedAt": "2026-03-24T11:01:25.200Z", + "baseModelName": null }, "gemini-2.5-flash-lite": { - provider: "google", - description: - "Google's most cost-efficient Gemini model, optimized for low-latency use cases with strong reasoning, multilingual, and long-context capabilities at minimal cost.", - contextWindow: 1048576, - maxOutputTokens: 65535, - capabilities: [ + "provider": "google", + "description": "Google's most cost-efficient Gemini model, optimized for low-latency use cases with strong reasoning, multilingual, and long-context capabilities at minimal cost.", + "contextWindow": 1048576, + "maxOutputTokens": 65535, + "capabilities": [ "vision", "tool_use", "streaming", "json_mode", "extended_thinking", "code_execution", - "audio_input", + "audio_input" ], - releaseDate: "2025-07-22", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2026-07-22", - knowledgeCutoff: "2025-01-01", - resolvedAt: "2026-03-24T11:02:30.060Z", - baseModelName: null, + "releaseDate": "2025-07-22", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2026-07-22", + "knowledgeCutoff": "2025-01-01", + "resolvedAt": "2026-03-24T11:02:30.060Z", + "baseModelName": null }, "gemini-2.5-pro": { - provider: "google", - description: - "Google's most advanced reasoning model with deep thinking capabilities, excelling at complex tasks like coding, math, and multimodal understanding across text, images, audio, and video.", - contextWindow: 1048576, - maxOutputTokens: 65535, - capabilities: [ + "provider": "google", + "description": "Google's most advanced reasoning model with deep thinking capabilities, excelling at complex tasks like coding, math, and multimodal understanding across text, images, audio, and video.", + "contextWindow": 1048576, + "maxOutputTokens": 65535, + "capabilities": [ "vision", "tool_use", "streaming", "json_mode", "extended_thinking", "code_execution", - "audio_input", + "audio_input" ], - releaseDate: "2025-03-25", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2026-06-17", - knowledgeCutoff: "2025-01-31", - resolvedAt: "2026-03-24T11:02:25.573Z", - baseModelName: null, + "releaseDate": "2025-03-25", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2026-06-17", + "knowledgeCutoff": "2025-01-31", + "resolvedAt": "2026-03-24T11:02:25.573Z", + "baseModelName": null }, "gemini-3-flash-preview": { - provider: "google", - description: - "Google's high-speed thinking model that matches Gemini 2.5 Pro performance at ~3x faster speed and lower cost, designed for agentic workflows, multi-turn chat, and coding assistance with configurable reasoning levels.", - contextWindow: 1048576, - maxOutputTokens: 65536, - capabilities: [ + "provider": "google", + "description": "Google's high-speed thinking model that matches Gemini 2.5 Pro performance at ~3x faster speed and lower cost, designed for agentic workflows, multi-turn chat, and coding assistance with configurable reasoning levels.", + "contextWindow": 1048576, + "maxOutputTokens": 65536, + "capabilities": [ "vision", "tool_use", "streaming", "json_mode", "extended_thinking", "code_execution", - "audio_input", + "audio_input" ], - releaseDate: "2025-12-17", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-01-01", - resolvedAt: "2026-03-24T11:02:13.388Z", - baseModelName: null, + "releaseDate": "2025-12-17", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-01-01", + "resolvedAt": "2026-03-24T11:02:13.388Z", + "baseModelName": null }, "gemini-3-pro-preview": { - provider: "google", - description: - "Google's flagship reasoning and multimodal model with strong coding and agentic capabilities, now deprecated in favor of Gemini 3.1 Pro.", - contextWindow: 1048576, - maxOutputTokens: 65536, - capabilities: [ + "provider": "google", + "description": "Google's flagship reasoning and multimodal model with strong coding and agentic capabilities, now deprecated in favor of Gemini 3.1 Pro.", + "contextWindow": 1048576, + "maxOutputTokens": 65536, + "capabilities": [ "vision", "tool_use", "streaming", "json_mode", "extended_thinking", "code_execution", - "audio_input", + "audio_input" ], - releaseDate: "2025-11-01", - isHidden: true, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2026-03-09", - knowledgeCutoff: "2025-01-01", - resolvedAt: "2026-03-24T11:02:29.313Z", - baseModelName: null, + "releaseDate": "2025-11-01", + "isHidden": true, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2026-03-09", + "knowledgeCutoff": "2025-01-01", + "resolvedAt": "2026-03-24T11:02:29.313Z", + "baseModelName": null }, "gemini-3.1-flash-lite-preview": { - provider: "google", - description: - "Google's most cost-efficient multimodal model in the Gemini 3 series, optimized for high-volume, low-latency tasks like translation, classification, and simple data extraction. Offers 2.5x faster time-to-first-token than Gemini 2.5 Flash.", - contextWindow: 1048576, - maxOutputTokens: 65536, - capabilities: [ + "provider": "google", + "description": "Google's most cost-efficient multimodal model in the Gemini 3 series, optimized for high-volume, low-latency tasks like translation, classification, and simple data extraction. Offers 2.5x faster time-to-first-token than Gemini 2.5 Flash.", + "contextWindow": 1048576, + "maxOutputTokens": 65536, + "capabilities": [ "vision", "tool_use", "streaming", "json_mode", "extended_thinking", "code_execution", - "audio_input", + "audio_input" ], - releaseDate: "2026-03-03", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-01-01", - resolvedAt: "2026-03-24T11:02:29.253Z", - baseModelName: null, + "releaseDate": "2026-03-03", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-01-01", + "resolvedAt": "2026-03-24T11:02:29.253Z", + "baseModelName": null }, "gemini-3.1-pro-preview": { - provider: "google", - description: - "Google's most advanced reasoning model in the Gemini 3.1 family, excelling at complex problem-solving across text, audio, images, video, and code with a 1M token context window and extended thinking capabilities.", - contextWindow: 1048576, - maxOutputTokens: 65536, - capabilities: [ + "provider": "google", + "description": "Google's most advanced reasoning model in the Gemini 3.1 family, excelling at complex problem-solving across text, audio, images, video, and code with a 1M token context window and extended thinking capabilities.", + "contextWindow": 1048576, + "maxOutputTokens": 65536, + "capabilities": [ "vision", "tool_use", "streaming", "json_mode", "extended_thinking", "code_execution", - "audio_input", + "audio_input" ], - releaseDate: "2026-02-19", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-01-01", - resolvedAt: "2026-03-24T11:03:33.071Z", - baseModelName: null, + "releaseDate": "2026-02-19", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-01-01", + "resolvedAt": "2026-03-24T11:03:33.071Z", + "baseModelName": null }, "gemini-pro": { - provider: "google", - description: - "Google's first-generation Gemini model for text generation, reasoning, and multi-turn conversation. Superseded by Gemini 1.5 Pro and later models.", - contextWindow: 32768, - maxOutputTokens: 8192, - capabilities: ["tool_use", "streaming", "json_mode"], - releaseDate: "2023-12-13", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: false, - deprecationDate: "2025-04-09", - knowledgeCutoff: "2023-04-01", - resolvedAt: "2026-03-24T11:03:45.401Z", - baseModelName: null, + "provider": "google", + "description": "Google's first-generation Gemini model for text generation, reasoning, and multi-turn conversation. Superseded by Gemini 1.5 Pro and later models.", + "contextWindow": 32768, + "maxOutputTokens": 8192, + "capabilities": [ + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2023-12-13", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": false, + "deprecationDate": "2025-04-09", + "knowledgeCutoff": "2023-04-01", + "resolvedAt": "2026-03-24T11:03:45.401Z", + "baseModelName": null }, "gpt-3.5-turbo": { - provider: "openai", - description: - "OpenAI's fast and cost-effective model optimized for chat and instruction-following tasks, now superseded by GPT-4o mini.", - contextWindow: 16385, - maxOutputTokens: 4096, - capabilities: ["tool_use", "streaming", "json_mode", "fine_tunable"], - releaseDate: "2023-03-01", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2025-09-13", - knowledgeCutoff: "2021-09-01", - resolvedAt: "2026-03-24T11:03:11.412Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's fast and cost-effective model optimized for chat and instruction-following tasks, now superseded by GPT-4o mini.", + "contextWindow": 16385, + "maxOutputTokens": 4096, + "capabilities": [ + "tool_use", + "streaming", + "json_mode", + "fine_tunable" + ], + "releaseDate": "2023-03-01", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2025-09-13", + "knowledgeCutoff": "2021-09-01", + "resolvedAt": "2026-03-24T11:03:11.412Z", + "baseModelName": null }, "gpt-3.5-turbo-0125": { - provider: "openai", - description: - "A fast and cost-effective GPT-3.5 Turbo snapshot optimized for chat completions, offering improved accuracy for function calling and reduced instances of incomplete responses.", - contextWindow: 16385, - maxOutputTokens: 4096, - capabilities: ["tool_use", "streaming", "json_mode", "fine_tunable"], - releaseDate: "2024-01-25", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2025-09-13", - knowledgeCutoff: "2021-09-01", - resolvedAt: "2026-03-24T11:03:11.310Z", - baseModelName: null, + "provider": "openai", + "description": "A fast and cost-effective GPT-3.5 Turbo snapshot optimized for chat completions, offering improved accuracy for function calling and reduced instances of incomplete responses.", + "contextWindow": 16385, + "maxOutputTokens": 4096, + "capabilities": [ + "tool_use", + "streaming", + "json_mode", + "fine_tunable" + ], + "releaseDate": "2024-01-25", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2025-09-13", + "knowledgeCutoff": "2021-09-01", + "resolvedAt": "2026-03-24T11:03:11.310Z", + "baseModelName": null }, "gpt-3.5-turbo-0301": { - provider: "openai", - description: - "Early snapshot of GPT-3.5 Turbo, OpenAI's first ChatGPT-optimized model for chat completions. Fast and cost-effective for simple tasks but superseded by later revisions.", - contextWindow: 4096, - maxOutputTokens: 4096, - capabilities: ["streaming", "fine_tunable"], - releaseDate: "2023-03-01", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: "2024-06-13", - knowledgeCutoff: "2021-09-01", - resolvedAt: "2026-03-24T11:03:12.060Z", - baseModelName: null, + "provider": "openai", + "description": "Early snapshot of GPT-3.5 Turbo, OpenAI's first ChatGPT-optimized model for chat completions. Fast and cost-effective for simple tasks but superseded by later revisions.", + "contextWindow": 4096, + "maxOutputTokens": 4096, + "capabilities": [ + "streaming", + "fine_tunable" + ], + "releaseDate": "2023-03-01", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": "2024-06-13", + "knowledgeCutoff": "2021-09-01", + "resolvedAt": "2026-03-24T11:03:12.060Z", + "baseModelName": null }, "gpt-3.5-turbo-0613": { - provider: "openai", - description: - "A snapshot of GPT-3.5 Turbo from June 2023, optimized for chat and instruction-following tasks with function calling support.", - contextWindow: 4096, - maxOutputTokens: 4096, - capabilities: ["tool_use", "streaming", "json_mode", "fine_tunable"], - releaseDate: "2023-06-13", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: "2024-09-13", - knowledgeCutoff: "2021-09-01", - resolvedAt: "2026-03-24T11:04:04.463Z", - baseModelName: null, + "provider": "openai", + "description": "A snapshot of GPT-3.5 Turbo from June 2023, optimized for chat and instruction-following tasks with function calling support.", + "contextWindow": 4096, + "maxOutputTokens": 4096, + "capabilities": [ + "tool_use", + "streaming", + "json_mode", + "fine_tunable" + ], + "releaseDate": "2023-06-13", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": "2024-09-13", + "knowledgeCutoff": "2021-09-01", + "resolvedAt": "2026-03-24T11:04:04.463Z", + "baseModelName": null }, "gpt-3.5-turbo-1106": { - provider: "openai", - description: - "A dated snapshot of GPT-3.5 Turbo released in November 2023, offering improved instruction following, JSON mode, and parallel function calling over previous GPT-3.5 variants.", - contextWindow: 16385, - maxOutputTokens: 4096, - capabilities: ["tool_use", "streaming", "json_mode", "fine_tunable"], - releaseDate: "2023-11-06", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2025-09-13", - knowledgeCutoff: "2021-09-01", - resolvedAt: "2026-03-24T11:04:23.054Z", - baseModelName: null, + "provider": "openai", + "description": "A dated snapshot of GPT-3.5 Turbo released in November 2023, offering improved instruction following, JSON mode, and parallel function calling over previous GPT-3.5 variants.", + "contextWindow": 16385, + "maxOutputTokens": 4096, + "capabilities": [ + "tool_use", + "streaming", + "json_mode", + "fine_tunable" + ], + "releaseDate": "2023-11-06", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2025-09-13", + "knowledgeCutoff": "2021-09-01", + "resolvedAt": "2026-03-24T11:04:23.054Z", + "baseModelName": null }, "gpt-3.5-turbo-16k": { - provider: "openai", - description: - "Extended context version of GPT-3.5 Turbo with 16K token context window, offering the same capabilities as the base model but able to process longer inputs.", - contextWindow: 16384, - maxOutputTokens: 4096, - capabilities: ["streaming", "json_mode", "fine_tunable", "tool_use"], - releaseDate: "2023-06-13", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2025-09-13", - knowledgeCutoff: "2021-09-01", - resolvedAt: "2026-03-24T11:04:36.307Z", - baseModelName: null, + "provider": "openai", + "description": "Extended context version of GPT-3.5 Turbo with 16K token context window, offering the same capabilities as the base model but able to process longer inputs.", + "contextWindow": 16384, + "maxOutputTokens": 4096, + "capabilities": [ + "streaming", + "json_mode", + "fine_tunable", + "tool_use" + ], + "releaseDate": "2023-06-13", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2025-09-13", + "knowledgeCutoff": "2021-09-01", + "resolvedAt": "2026-03-24T11:04:36.307Z", + "baseModelName": null }, "gpt-3.5-turbo-16k-0613": { - provider: "openai", - description: - "Extended context window variant of GPT-3.5 Turbo with 16K token context, snapshot from June 2023. Optimized for chat completions with longer document processing.", - contextWindow: 16384, - maxOutputTokens: 4096, - capabilities: ["streaming", "json_mode", "fine_tunable"], - releaseDate: "2023-06-13", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: "2024-09-13", - knowledgeCutoff: "2021-09-01", - resolvedAt: "2026-03-24T11:04:22.894Z", - baseModelName: null, + "provider": "openai", + "description": "Extended context window variant of GPT-3.5 Turbo with 16K token context, snapshot from June 2023. Optimized for chat completions with longer document processing.", + "contextWindow": 16384, + "maxOutputTokens": 4096, + "capabilities": [ + "streaming", + "json_mode", + "fine_tunable" + ], + "releaseDate": "2023-06-13", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": "2024-09-13", + "knowledgeCutoff": "2021-09-01", + "resolvedAt": "2026-03-24T11:04:22.894Z", + "baseModelName": null }, "gpt-3.5-turbo-instruct": { - provider: "openai", - description: - "OpenAI's GPT-3.5 Turbo Instruct is a completions-only model (not chat) optimized for following explicit instructions, replacing the legacy text-davinci-003 model.", - contextWindow: 4096, - maxOutputTokens: 4096, - capabilities: ["fine_tunable"], - releaseDate: "2023-09-19", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: "2025-01-27", - knowledgeCutoff: "2021-09-01", - resolvedAt: "2026-03-24T11:04:22.309Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's GPT-3.5 Turbo Instruct is a completions-only model (not chat) optimized for following explicit instructions, replacing the legacy text-davinci-003 model.", + "contextWindow": 4096, + "maxOutputTokens": 4096, + "capabilities": [ + "fine_tunable" + ], + "releaseDate": "2023-09-19", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": "2025-01-27", + "knowledgeCutoff": "2021-09-01", + "resolvedAt": "2026-03-24T11:04:22.309Z", + "baseModelName": null }, "gpt-4": { - provider: "openai", - description: - "OpenAI's flagship large language model that preceded GPT-4o, known for strong reasoning and instruction-following capabilities across a wide range of tasks.", - contextWindow: 8192, - maxOutputTokens: 8192, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2023-03-14", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2025-06-06", - knowledgeCutoff: "2023-12-01", - resolvedAt: "2026-03-24T11:04:36.773Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's flagship large language model that preceded GPT-4o, known for strong reasoning and instruction-following capabilities across a wide range of tasks.", + "contextWindow": 8192, + "maxOutputTokens": 8192, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2023-03-14", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2025-06-06", + "knowledgeCutoff": "2023-12-01", + "resolvedAt": "2026-03-24T11:04:36.773Z", + "baseModelName": null }, "gpt-4-0125-preview": { - provider: "openai", - description: - "An improved GPT-4 Turbo preview model with better task completion, reduced laziness in code generation, and enhanced instruction following.", - contextWindow: 128000, - maxOutputTokens: 4096, - capabilities: ["tool_use", "streaming", "json_mode"], - releaseDate: "2024-01-25", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2023-12-01", - resolvedAt: "2026-03-24T11:04:54.196Z", - baseModelName: null, + "provider": "openai", + "description": "An improved GPT-4 Turbo preview model with better task completion, reduced laziness in code generation, and enhanced instruction following.", + "contextWindow": 128000, + "maxOutputTokens": 4096, + "capabilities": [ + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2024-01-25", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2023-12-01", + "resolvedAt": "2026-03-24T11:04:54.196Z", + "baseModelName": null }, "gpt-4-0314": { - provider: "openai", - description: - "Original GPT-4 snapshot from March 2023, a large multimodal model (text-only at launch) that was one of OpenAI's first GPT-4 releases. Now deprecated and replaced by newer GPT-4 variants.", - contextWindow: 8192, - maxOutputTokens: 4096, - capabilities: ["streaming"], - releaseDate: "2023-03-14", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: "2024-06-13", - knowledgeCutoff: "2021-09-01", - resolvedAt: "2026-03-24T11:05:14.112Z", - baseModelName: null, + "provider": "openai", + "description": "Original GPT-4 snapshot from March 2023, a large multimodal model (text-only at launch) that was one of OpenAI's first GPT-4 releases. Now deprecated and replaced by newer GPT-4 variants.", + "contextWindow": 8192, + "maxOutputTokens": 4096, + "capabilities": [ + "streaming" + ], + "releaseDate": "2023-03-14", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": "2024-06-13", + "knowledgeCutoff": "2021-09-01", + "resolvedAt": "2026-03-24T11:05:14.112Z", + "baseModelName": null }, "gpt-4-0613": { - provider: "openai", - description: - "A snapshot of GPT-4 from June 2023, offering strong reasoning and instruction-following capabilities. It was one of the first widely available GPT-4 variants with function calling support.", - contextWindow: 8192, - maxOutputTokens: 8192, - capabilities: ["tool_use", "streaming", "json_mode", "fine_tunable"], - releaseDate: "2023-06-13", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: "2025-06-06", - knowledgeCutoff: "2021-09-01", - resolvedAt: "2026-03-24T11:05:13.885Z", - baseModelName: null, + "provider": "openai", + "description": "A snapshot of GPT-4 from June 2023, offering strong reasoning and instruction-following capabilities. It was one of the first widely available GPT-4 variants with function calling support.", + "contextWindow": 8192, + "maxOutputTokens": 8192, + "capabilities": [ + "tool_use", + "streaming", + "json_mode", + "fine_tunable" + ], + "releaseDate": "2023-06-13", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": "2025-06-06", + "knowledgeCutoff": "2021-09-01", + "resolvedAt": "2026-03-24T11:05:13.885Z", + "baseModelName": null }, "gpt-4-1106-preview": { - provider: "openai", - description: - "GPT-4 Turbo preview model with 128K context window, offering improved instruction following and JSON mode support at reduced cost compared to GPT-4.", - contextWindow: 128000, - maxOutputTokens: 4096, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2023-11-06", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2023-04-01", - resolvedAt: "2026-03-24T11:05:12.960Z", - baseModelName: null, + "provider": "openai", + "description": "GPT-4 Turbo preview model with 128K context window, offering improved instruction following and JSON mode support at reduced cost compared to GPT-4.", + "contextWindow": 128000, + "maxOutputTokens": 4096, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2023-11-06", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2023-04-01", + "resolvedAt": "2026-03-24T11:05:12.960Z", + "baseModelName": null }, "gpt-4-32k": { - provider: "openai", - description: - "Extended context window variant of GPT-4 with 32,768 token capacity, offering the same capabilities as GPT-4 but able to process longer documents and conversations.", - contextWindow: 32768, - maxOutputTokens: 4096, - capabilities: ["tool_use", "streaming", "json_mode"], - releaseDate: "2023-03-14", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: "2025-06-06", - knowledgeCutoff: "2021-09-01", - resolvedAt: "2026-03-24T11:05:14.584Z", - baseModelName: null, + "provider": "openai", + "description": "Extended context window variant of GPT-4 with 32,768 token capacity, offering the same capabilities as GPT-4 but able to process longer documents and conversations.", + "contextWindow": 32768, + "maxOutputTokens": 4096, + "capabilities": [ + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2023-03-14", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": "2025-06-06", + "knowledgeCutoff": "2021-09-01", + "resolvedAt": "2026-03-24T11:05:14.584Z", + "baseModelName": null }, "gpt-4-32k-0314": { - provider: "openai", - description: - "Extended context (32k token) variant of the original GPT-4 launch snapshot from March 2024, offering the same capabilities as gpt-4-0314 but with 4x the context window.", - contextWindow: 32768, - maxOutputTokens: 4096, - capabilities: ["streaming"], - releaseDate: "2023-03-14", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: "2024-06-13", - knowledgeCutoff: "2021-09-01", - resolvedAt: "2026-03-24T11:05:32.044Z", - baseModelName: null, + "provider": "openai", + "description": "Extended context (32k token) variant of the original GPT-4 launch snapshot from March 2024, offering the same capabilities as gpt-4-0314 but with 4x the context window.", + "contextWindow": 32768, + "maxOutputTokens": 4096, + "capabilities": [ + "streaming" + ], + "releaseDate": "2023-03-14", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": "2024-06-13", + "knowledgeCutoff": "2021-09-01", + "resolvedAt": "2026-03-24T11:05:32.044Z", + "baseModelName": null }, "gpt-4-32k-0613": { - provider: "openai", - description: - "Extended context window variant of GPT-4 with 32,768 token context, based on the June 2023 snapshot. Offers the same capabilities as GPT-4 but with 4x the context length.", - contextWindow: 32768, - maxOutputTokens: 4096, - capabilities: ["tool_use", "streaming", "json_mode"], - releaseDate: "2023-06-13", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: "2025-06-06", - knowledgeCutoff: "2021-09-01", - resolvedAt: "2026-03-24T11:05:53.070Z", - baseModelName: null, + "provider": "openai", + "description": "Extended context window variant of GPT-4 with 32,768 token context, based on the June 2023 snapshot. Offers the same capabilities as GPT-4 but with 4x the context length.", + "contextWindow": 32768, + "maxOutputTokens": 4096, + "capabilities": [ + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2023-06-13", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": "2025-06-06", + "knowledgeCutoff": "2021-09-01", + "resolvedAt": "2026-03-24T11:05:53.070Z", + "baseModelName": null }, "gpt-4-preview": { - provider: "openai", - description: - "GPT-4 Turbo preview model with 128K context window, JSON mode, and parallel function calling. A preview release in the GPT-4 Turbo series, now deprecated in favor of newer models.", - contextWindow: 128000, - maxOutputTokens: 4096, - capabilities: ["tool_use", "streaming", "json_mode"], - releaseDate: "2023-11-06", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2025-06-06", - knowledgeCutoff: "2023-04-01", - resolvedAt: "2026-03-24T11:06:54.248Z", - baseModelName: null, + "provider": "openai", + "description": "GPT-4 Turbo preview model with 128K context window, JSON mode, and parallel function calling. A preview release in the GPT-4 Turbo series, now deprecated in favor of newer models.", + "contextWindow": 128000, + "maxOutputTokens": 4096, + "capabilities": [ + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2023-11-06", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2025-06-06", + "knowledgeCutoff": "2023-04-01", + "resolvedAt": "2026-03-24T11:06:54.248Z", + "baseModelName": null }, "gpt-4-turbo": { - provider: "openai", - description: - "OpenAI's optimized GPT-4 variant offering faster inference and lower cost than the original GPT-4, with vision capabilities and a 128K context window.", - contextWindow: 128000, - maxOutputTokens: 4096, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2024-04-09", - isHidden: false, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2023-12-01", - resolvedAt: "2026-03-24T11:05:51.415Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's optimized GPT-4 variant offering faster inference and lower cost than the original GPT-4, with vision capabilities and a 128K context window.", + "contextWindow": 128000, + "maxOutputTokens": 4096, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2024-04-09", + "isHidden": false, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2023-12-01", + "resolvedAt": "2026-03-24T11:05:51.415Z", + "baseModelName": null }, "gpt-4-turbo-2024-04-09": { - provider: "openai", - description: - "OpenAI's optimized GPT-4 variant offering faster inference and lower cost than the original GPT-4, with vision capabilities and a 128K context window.", - contextWindow: 128000, - maxOutputTokens: 4096, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2024-04-09", - isHidden: false, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2023-12-01", - resolvedAt: "2026-03-24T11:05:51.415Z", - baseModelName: "gpt-4-turbo", + "provider": "openai", + "description": "OpenAI's optimized GPT-4 variant offering faster inference and lower cost than the original GPT-4, with vision capabilities and a 128K context window.", + "contextWindow": 128000, + "maxOutputTokens": 4096, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2024-04-09", + "isHidden": false, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2023-12-01", + "resolvedAt": "2026-03-24T11:05:51.415Z", + "baseModelName": "gpt-4-turbo" }, "gpt-4-turbo-preview": { - provider: "openai", - description: - "An early preview of GPT-4 Turbo with a 128K context window, offering improved instruction following and JSON mode support at reduced cost compared to GPT-4.", - contextWindow: 128000, - maxOutputTokens: 4096, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2024-01-25", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2023-12-01", - resolvedAt: "2026-03-24T11:05:52.346Z", - baseModelName: null, + "provider": "openai", + "description": "An early preview of GPT-4 Turbo with a 128K context window, offering improved instruction following and JSON mode support at reduced cost compared to GPT-4.", + "contextWindow": 128000, + "maxOutputTokens": 4096, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2024-01-25", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2023-12-01", + "resolvedAt": "2026-03-24T11:05:52.346Z", + "baseModelName": null }, "gpt-4-turbo-vision": { - provider: "openai", - description: - "OpenAI's GPT-4 Turbo model with vision capabilities, able to analyze and understand images alongside text. It was a preview model later superseded by GPT-4 Turbo (gpt-4-turbo-2024-04-09) and then GPT-4o.", - contextWindow: 128000, - maxOutputTokens: 4096, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2023-11-06", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2024-12-06", - knowledgeCutoff: "2023-04-01", - resolvedAt: "2026-03-24T11:06:38.455Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's GPT-4 Turbo model with vision capabilities, able to analyze and understand images alongside text. It was a preview model later superseded by GPT-4 Turbo (gpt-4-turbo-2024-04-09) and then GPT-4o.", + "contextWindow": 128000, + "maxOutputTokens": 4096, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2023-11-06", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2024-12-06", + "knowledgeCutoff": "2023-04-01", + "resolvedAt": "2026-03-24T11:06:38.455Z", + "baseModelName": null }, "gpt-4.1": { - provider: "openai", - description: - "OpenAI's flagship model optimized for coding, instruction following, and tool calling with a 1M token context window. Excels at structured outputs and long-context tasks.", - contextWindow: 1047576, - maxOutputTokens: 32768, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2025-04-14", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-06-01", - resolvedAt: "2026-03-24T11:07:00.439Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's flagship model optimized for coding, instruction following, and tool calling with a 1M token context window. Excels at structured outputs and long-context tasks.", + "contextWindow": 1047576, + "maxOutputTokens": 32768, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2025-04-14", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-06-01", + "resolvedAt": "2026-03-24T11:07:00.439Z", + "baseModelName": null }, "gpt-4.1-2025-04-14": { - provider: "openai", - description: - "OpenAI's flagship model optimized for coding, instruction following, and tool calling with a 1M token context window. Excels at structured outputs and long-context tasks.", - contextWindow: 1047576, - maxOutputTokens: 32768, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2025-04-14", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-06-01", - resolvedAt: "2026-03-24T11:07:00.439Z", - baseModelName: "gpt-4.1", + "provider": "openai", + "description": "OpenAI's flagship model optimized for coding, instruction following, and tool calling with a 1M token context window. Excels at structured outputs and long-context tasks.", + "contextWindow": 1047576, + "maxOutputTokens": 32768, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2025-04-14", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-06-01", + "resolvedAt": "2026-03-24T11:07:00.439Z", + "baseModelName": "gpt-4.1" }, "gpt-4.1-mini": { - provider: "openai", - description: - "A compact, cost-efficient model in OpenAI's GPT-4.1 family that matches or exceeds GPT-4o on many benchmarks while offering nearly half the latency and significantly lower cost.", - contextWindow: 1000000, - maxOutputTokens: 32768, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "fine_tunable"], - releaseDate: "2025-04-14", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-06-01", - resolvedAt: "2026-03-24T11:08:14.524Z", - baseModelName: null, + "provider": "openai", + "description": "A compact, cost-efficient model in OpenAI's GPT-4.1 family that matches or exceeds GPT-4o on many benchmarks while offering nearly half the latency and significantly lower cost.", + "contextWindow": 1000000, + "maxOutputTokens": 32768, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "fine_tunable" + ], + "releaseDate": "2025-04-14", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-06-01", + "resolvedAt": "2026-03-24T11:08:14.524Z", + "baseModelName": null }, "gpt-4.1-mini-2025-04-14": { - provider: "openai", - description: - "A compact, cost-efficient model in OpenAI's GPT-4.1 family that matches or exceeds GPT-4o on many benchmarks while offering nearly half the latency and significantly lower cost.", - contextWindow: 1000000, - maxOutputTokens: 32768, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "fine_tunable"], - releaseDate: "2025-04-14", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-06-01", - resolvedAt: "2026-03-24T11:08:14.524Z", - baseModelName: "gpt-4.1-mini", + "provider": "openai", + "description": "A compact, cost-efficient model in OpenAI's GPT-4.1 family that matches or exceeds GPT-4o on many benchmarks while offering nearly half the latency and significantly lower cost.", + "contextWindow": 1000000, + "maxOutputTokens": 32768, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "fine_tunable" + ], + "releaseDate": "2025-04-14", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-06-01", + "resolvedAt": "2026-03-24T11:08:14.524Z", + "baseModelName": "gpt-4.1-mini" }, "gpt-4.1-nano": { - provider: "openai", - description: - "OpenAI's fastest and most cost-effective model in the GPT-4.1 family, optimized for low-latency tasks like classification, autocompletion, and lightweight agentic workflows with strong instruction-following and tool-calling capabilities.", - contextWindow: 1047576, - maxOutputTokens: 32768, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2025-04-14", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-06-01", - resolvedAt: "2026-03-24T11:08:04.533Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's fastest and most cost-effective model in the GPT-4.1 family, optimized for low-latency tasks like classification, autocompletion, and lightweight agentic workflows with strong instruction-following and tool-calling capabilities.", + "contextWindow": 1047576, + "maxOutputTokens": 32768, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2025-04-14", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-06-01", + "resolvedAt": "2026-03-24T11:08:04.533Z", + "baseModelName": null }, "gpt-4.1-nano-2025-04-14": { - provider: "openai", - description: - "OpenAI's fastest and most cost-effective model in the GPT-4.1 family, optimized for low-latency tasks like classification, autocompletion, and lightweight agentic workflows with strong instruction-following and tool-calling capabilities.", - contextWindow: 1047576, - maxOutputTokens: 32768, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2025-04-14", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-06-01", - resolvedAt: "2026-03-24T11:08:04.533Z", - baseModelName: "gpt-4.1-nano", + "provider": "openai", + "description": "OpenAI's fastest and most cost-effective model in the GPT-4.1 family, optimized for low-latency tasks like classification, autocompletion, and lightweight agentic workflows with strong instruction-following and tool-calling capabilities.", + "contextWindow": 1047576, + "maxOutputTokens": 32768, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2025-04-14", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-06-01", + "resolvedAt": "2026-03-24T11:08:04.533Z", + "baseModelName": "gpt-4.1-nano" }, "gpt-4.5-preview": { - provider: "openai", - description: - "OpenAI's largest pretrained model before the GPT-5 series, emphasizing broad knowledge, creative writing, and improved emotional intelligence over reasoning-focused models.", - contextWindow: 128000, - maxOutputTokens: 16384, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2025-02-27", - isHidden: true, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2025-07-14", - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:07:57.880Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's largest pretrained model before the GPT-5 series, emphasizing broad knowledge, creative writing, and improved emotional intelligence over reasoning-focused models.", + "contextWindow": 128000, + "maxOutputTokens": 16384, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2025-02-27", + "isHidden": true, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2025-07-14", + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:07:57.880Z", + "baseModelName": null }, "gpt-4.5-preview-2025-02-27": { - provider: "openai", - description: - "OpenAI's largest pretrained model before the GPT-5 series, emphasizing broad knowledge, creative writing, and improved emotional intelligence over reasoning-focused models.", - contextWindow: 128000, - maxOutputTokens: 16384, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2025-02-27", - isHidden: true, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2025-07-14", - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:07:57.880Z", - baseModelName: "gpt-4.5-preview", + "provider": "openai", + "description": "OpenAI's largest pretrained model before the GPT-5 series, emphasizing broad knowledge, creative writing, and improved emotional intelligence over reasoning-focused models.", + "contextWindow": 128000, + "maxOutputTokens": 16384, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2025-02-27", + "isHidden": true, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2025-07-14", + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:07:57.880Z", + "baseModelName": "gpt-4.5-preview" }, "gpt-4o": { - provider: "openai", - description: - "OpenAI's flagship multimodal model combining strong reasoning with vision, audio, and tool use capabilities at faster speeds and lower cost than GPT-4 Turbo.", - contextWindow: 128000, - maxOutputTokens: 16384, - capabilities: [ + "provider": "openai", + "description": "OpenAI's flagship multimodal model combining strong reasoning with vision, audio, and tool use capabilities at faster speeds and lower cost than GPT-4 Turbo.", + "contextWindow": 128000, + "maxOutputTokens": 16384, + "capabilities": [ "vision", "tool_use", "streaming", "json_mode", "audio_input", "audio_output", - "fine_tunable", + "fine_tunable" ], - releaseDate: "2024-05-13", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:07:31.638Z", - baseModelName: null, + "releaseDate": "2024-05-13", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:07:31.638Z", + "baseModelName": null }, "gpt-4o-2024-05-13": { - provider: "openai", - description: - "OpenAI's flagship multimodal model combining strong reasoning with vision, audio, and tool use capabilities at faster speeds and lower cost than GPT-4 Turbo.", - contextWindow: 128000, - maxOutputTokens: 16384, - capabilities: [ + "provider": "openai", + "description": "OpenAI's flagship multimodal model combining strong reasoning with vision, audio, and tool use capabilities at faster speeds and lower cost than GPT-4 Turbo.", + "contextWindow": 128000, + "maxOutputTokens": 16384, + "capabilities": [ "vision", "tool_use", "streaming", "json_mode", "audio_input", "audio_output", - "fine_tunable", + "fine_tunable" ], - releaseDate: "2024-05-13", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:07:31.638Z", - baseModelName: "gpt-4o", + "releaseDate": "2024-05-13", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:07:31.638Z", + "baseModelName": "gpt-4o" }, "gpt-4o-2024-08-06": { - provider: "openai", - description: - "OpenAI's flagship multimodal model combining strong reasoning with vision, audio, and tool use capabilities at faster speeds and lower cost than GPT-4 Turbo.", - contextWindow: 128000, - maxOutputTokens: 16384, - capabilities: [ + "provider": "openai", + "description": "OpenAI's flagship multimodal model combining strong reasoning with vision, audio, and tool use capabilities at faster speeds and lower cost than GPT-4 Turbo.", + "contextWindow": 128000, + "maxOutputTokens": 16384, + "capabilities": [ "vision", "tool_use", "streaming", "json_mode", "audio_input", "audio_output", - "fine_tunable", + "fine_tunable" ], - releaseDate: "2024-05-13", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:07:31.638Z", - baseModelName: "gpt-4o", + "releaseDate": "2024-05-13", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:07:31.638Z", + "baseModelName": "gpt-4o" }, "gpt-4o-2024-11-20": { - provider: "openai", - description: - "OpenAI's flagship multimodal model combining strong reasoning with vision, audio, and tool use capabilities at faster speeds and lower cost than GPT-4 Turbo.", - contextWindow: 128000, - maxOutputTokens: 16384, - capabilities: [ + "provider": "openai", + "description": "OpenAI's flagship multimodal model combining strong reasoning with vision, audio, and tool use capabilities at faster speeds and lower cost than GPT-4 Turbo.", + "contextWindow": 128000, + "maxOutputTokens": 16384, + "capabilities": [ "vision", "tool_use", "streaming", "json_mode", "audio_input", "audio_output", - "fine_tunable", + "fine_tunable" ], - releaseDate: "2024-05-13", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:07:31.638Z", - baseModelName: "gpt-4o", + "releaseDate": "2024-05-13", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:07:31.638Z", + "baseModelName": "gpt-4o" }, "gpt-4o-audio-preview": { - provider: "openai", - description: - "GPT-4o variant with native audio input and output capabilities via the Chat Completions API, supporting both text and audio modalities for conversational and voice-based applications.", - contextWindow: 128000, - maxOutputTokens: 16384, - capabilities: ["audio_input", "audio_output", "tool_use", "streaming"], - releaseDate: "2024-10-01", - isHidden: false, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2026-05-07", - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:08:09.590Z", - baseModelName: null, + "provider": "openai", + "description": "GPT-4o variant with native audio input and output capabilities via the Chat Completions API, supporting both text and audio modalities for conversational and voice-based applications.", + "contextWindow": 128000, + "maxOutputTokens": 16384, + "capabilities": [ + "audio_input", + "audio_output", + "tool_use", + "streaming" + ], + "releaseDate": "2024-10-01", + "isHidden": false, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2026-05-07", + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:08:09.590Z", + "baseModelName": null }, "gpt-4o-audio-preview-2024-10-01": { - provider: "openai", - description: - "GPT-4o variant with native audio input and output capabilities via the Chat Completions API, supporting both text and audio modalities for conversational and voice-based applications.", - contextWindow: 128000, - maxOutputTokens: 16384, - capabilities: ["audio_input", "audio_output", "tool_use", "streaming"], - releaseDate: "2024-10-01", - isHidden: false, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: "2026-05-07", - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:08:09.590Z", - baseModelName: "gpt-4o-audio-preview", + "provider": "openai", + "description": "GPT-4o variant with native audio input and output capabilities via the Chat Completions API, supporting both text and audio modalities for conversational and voice-based applications.", + "contextWindow": 128000, + "maxOutputTokens": 16384, + "capabilities": [ + "audio_input", + "audio_output", + "tool_use", + "streaming" + ], + "releaseDate": "2024-10-01", + "isHidden": false, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": "2026-05-07", + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:08:09.590Z", + "baseModelName": "gpt-4o-audio-preview" }, "gpt-4o-mini": { - provider: "openai", - description: - "Fast, affordable small model optimized for focused tasks. Positioned as OpenAI's cost-efficient option with strong performance on benchmarks relative to its size.", - contextWindow: 128000, - maxOutputTokens: 16384, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "fine_tunable"], - releaseDate: "2024-07-18", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:09:50.130Z", - baseModelName: null, + "provider": "openai", + "description": "Fast, affordable small model optimized for focused tasks. Positioned as OpenAI's cost-efficient option with strong performance on benchmarks relative to its size.", + "contextWindow": 128000, + "maxOutputTokens": 16384, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "fine_tunable" + ], + "releaseDate": "2024-07-18", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:09:50.130Z", + "baseModelName": null }, "gpt-4o-mini-2024-07-18": { - provider: "openai", - description: - "Fast, affordable small model optimized for focused tasks. Positioned as OpenAI's cost-efficient option with strong performance on benchmarks relative to its size.", - contextWindow: 128000, - maxOutputTokens: 16384, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "fine_tunable"], - releaseDate: "2024-07-18", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:09:50.130Z", - baseModelName: "gpt-4o-mini", + "provider": "openai", + "description": "Fast, affordable small model optimized for focused tasks. Positioned as OpenAI's cost-efficient option with strong performance on benchmarks relative to its size.", + "contextWindow": 128000, + "maxOutputTokens": 16384, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "fine_tunable" + ], + "releaseDate": "2024-07-18", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:09:50.130Z", + "baseModelName": "gpt-4o-mini" }, "gpt-4o-realtime-preview": { - provider: "openai", - description: - "OpenAI's real-time multimodal model capable of processing and generating both text and audio over WebRTC or WebSocket, enabling low-latency voice conversations and audio interactions.", - contextWindow: 128000, - maxOutputTokens: 4096, - capabilities: ["audio_input", "audio_output", "tool_use", "streaming"], - releaseDate: "2024-10-01", - isHidden: false, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: false, - deprecationDate: "2026-05-07", - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:09:35.495Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's real-time multimodal model capable of processing and generating both text and audio over WebRTC or WebSocket, enabling low-latency voice conversations and audio interactions.", + "contextWindow": 128000, + "maxOutputTokens": 4096, + "capabilities": [ + "audio_input", + "audio_output", + "tool_use", + "streaming" + ], + "releaseDate": "2024-10-01", + "isHidden": false, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": false, + "deprecationDate": "2026-05-07", + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:09:35.495Z", + "baseModelName": null }, "gpt-4o-realtime-preview-2024-10-01": { - provider: "openai", - description: - "OpenAI's real-time multimodal model capable of processing and generating both text and audio over WebRTC or WebSocket, enabling low-latency voice conversations and audio interactions.", - contextWindow: 128000, - maxOutputTokens: 4096, - capabilities: ["audio_input", "audio_output", "tool_use", "streaming"], - releaseDate: "2024-10-01", - isHidden: false, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: false, - deprecationDate: "2026-05-07", - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:09:35.495Z", - baseModelName: "gpt-4o-realtime-preview", + "provider": "openai", + "description": "OpenAI's real-time multimodal model capable of processing and generating both text and audio over WebRTC or WebSocket, enabling low-latency voice conversations and audio interactions.", + "contextWindow": 128000, + "maxOutputTokens": 4096, + "capabilities": [ + "audio_input", + "audio_output", + "tool_use", + "streaming" + ], + "releaseDate": "2024-10-01", + "isHidden": false, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": false, + "deprecationDate": "2026-05-07", + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:09:35.495Z", + "baseModelName": "gpt-4o-realtime-preview" }, "gpt-5": { - provider: "openai", - description: - "OpenAI's flagship reasoning model released August 2025, featuring a 400K token context window with strong coding, reasoning, and agentic capabilities.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: [ + "provider": "openai", + "description": "OpenAI's flagship reasoning model released August 2025, featuring a 400K token context window with strong coding, reasoning, and agentic capabilities.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ "vision", "tool_use", "streaming", "json_mode", "image_generation", - "code_execution", + "code_execution" ], - releaseDate: "2025-08-07", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-09-30", - resolvedAt: "2026-03-24T11:09:28.216Z", - baseModelName: null, + "releaseDate": "2025-08-07", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-09-30", + "resolvedAt": "2026-03-24T11:09:28.216Z", + "baseModelName": null }, "gpt-5-2025-08-07": { - provider: "openai", - description: - "OpenAI's flagship reasoning model released August 2025, featuring a 400K token context window with strong coding, reasoning, and agentic capabilities.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: [ + "provider": "openai", + "description": "OpenAI's flagship reasoning model released August 2025, featuring a 400K token context window with strong coding, reasoning, and agentic capabilities.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ "vision", "tool_use", "streaming", "json_mode", "image_generation", - "code_execution", + "code_execution" ], - releaseDate: "2025-08-07", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-09-30", - resolvedAt: "2026-03-24T11:09:28.216Z", - baseModelName: "gpt-5", + "releaseDate": "2025-08-07", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-09-30", + "resolvedAt": "2026-03-24T11:09:28.216Z", + "baseModelName": "gpt-5" }, "gpt-5-chat-latest": { - provider: "openai", - description: - "Non-reasoning GPT-5 model used in ChatGPT, optimized for conversational tasks. Supports text and image inputs with function calling and structured outputs.", - contextWindow: 128000, - maxOutputTokens: 16384, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2025-08-07", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-09-30", - resolvedAt: "2026-03-24T11:09:24.834Z", - baseModelName: "gpt-5-chat", + "provider": "openai", + "description": "Non-reasoning GPT-5 model used in ChatGPT, optimized for conversational tasks. Supports text and image inputs with function calling and structured outputs.", + "contextWindow": 128000, + "maxOutputTokens": 16384, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2025-08-07", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-09-30", + "resolvedAt": "2026-03-24T11:09:24.834Z", + "baseModelName": "gpt-5-chat" }, "gpt-5-mini": { - provider: "openai", - description: - "A faster, more cost-efficient version of GPT-5 designed for well-defined tasks and precise prompts. Supports reasoning with configurable effort levels and offers reduced latency compared to the full GPT-5 model.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2025-08-07", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-05-31", - resolvedAt: "2026-03-24T11:09:42.822Z", - baseModelName: null, + "provider": "openai", + "description": "A faster, more cost-efficient version of GPT-5 designed for well-defined tasks and precise prompts. Supports reasoning with configurable effort levels and offers reduced latency compared to the full GPT-5 model.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2025-08-07", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-05-31", + "resolvedAt": "2026-03-24T11:09:42.822Z", + "baseModelName": null }, "gpt-5-mini-2025-08-07": { - provider: "openai", - description: - "A faster, more cost-efficient version of GPT-5 designed for well-defined tasks and precise prompts. Supports reasoning with configurable effort levels and offers reduced latency compared to the full GPT-5 model.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2025-08-07", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-05-31", - resolvedAt: "2026-03-24T11:09:42.822Z", - baseModelName: "gpt-5-mini", + "provider": "openai", + "description": "A faster, more cost-efficient version of GPT-5 designed for well-defined tasks and precise prompts. Supports reasoning with configurable effort levels and offers reduced latency compared to the full GPT-5 model.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2025-08-07", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-05-31", + "resolvedAt": "2026-03-24T11:09:42.822Z", + "baseModelName": "gpt-5-mini" }, "gpt-5-nano": { - provider: "openai", - description: - "The smallest and fastest variant in the GPT-5 family, optimized for developer tools, rapid interactions, and ultra-low latency environments. Best suited for classification, data extraction, ranking, and sub-agent tasks.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2025-08-07", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-05-31", - resolvedAt: "2026-03-24T11:11:24.884Z", - baseModelName: null, + "provider": "openai", + "description": "The smallest and fastest variant in the GPT-5 family, optimized for developer tools, rapid interactions, and ultra-low latency environments. Best suited for classification, data extraction, ranking, and sub-agent tasks.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2025-08-07", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-05-31", + "resolvedAt": "2026-03-24T11:11:24.884Z", + "baseModelName": null }, "gpt-5-nano-2025-08-07": { - provider: "openai", - description: - "The smallest and fastest variant in the GPT-5 family, optimized for developer tools, rapid interactions, and ultra-low latency environments. Best suited for classification, data extraction, ranking, and sub-agent tasks.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2025-08-07", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-05-31", - resolvedAt: "2026-03-24T11:11:24.884Z", - baseModelName: "gpt-5-nano", + "provider": "openai", + "description": "The smallest and fastest variant in the GPT-5 family, optimized for developer tools, rapid interactions, and ultra-low latency environments. Best suited for classification, data extraction, ranking, and sub-agent tasks.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2025-08-07", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-05-31", + "resolvedAt": "2026-03-24T11:11:24.884Z", + "baseModelName": "gpt-5-nano" }, "gpt-5-pro": { - provider: "openai", - description: - "OpenAI's enhanced GPT-5 variant optimized for complex tasks requiring step-by-step reasoning, with reduced hallucination and improved code quality compared to the base GPT-5.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2025-10-06", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-10-01", - resolvedAt: "2026-03-24T11:11:37.048Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's enhanced GPT-5 variant optimized for complex tasks requiring step-by-step reasoning, with reduced hallucination and improved code quality compared to the base GPT-5.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2025-10-06", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-10-01", + "resolvedAt": "2026-03-24T11:11:37.048Z", + "baseModelName": null }, "gpt-5-pro-2025-10-06": { - provider: "openai", - description: - "OpenAI's enhanced GPT-5 variant optimized for complex tasks requiring step-by-step reasoning, with reduced hallucination and improved code quality compared to the base GPT-5.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2025-10-06", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-10-01", - resolvedAt: "2026-03-24T11:11:37.048Z", - baseModelName: "gpt-5-pro", + "provider": "openai", + "description": "OpenAI's enhanced GPT-5 variant optimized for complex tasks requiring step-by-step reasoning, with reduced hallucination and improved code quality compared to the base GPT-5.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2025-10-06", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-10-01", + "resolvedAt": "2026-03-24T11:11:37.048Z", + "baseModelName": "gpt-5-pro" }, "gpt-5.1": { - provider: "openai", - description: - "GPT-5.1 is OpenAI's frontier-grade model in the GPT-5 series, offering adaptive reasoning with configurable effort levels, improved coding and math performance, and a more natural conversational style compared to GPT-5.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2025-11-13", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-09-30", - resolvedAt: "2026-03-24T11:11:47.327Z", - baseModelName: null, + "provider": "openai", + "description": "GPT-5.1 is OpenAI's frontier-grade model in the GPT-5 series, offering adaptive reasoning with configurable effort levels, improved coding and math performance, and a more natural conversational style compared to GPT-5.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2025-11-13", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-09-30", + "resolvedAt": "2026-03-24T11:11:47.327Z", + "baseModelName": null }, "gpt-5.1-2025-11-13": { - provider: "openai", - description: - "GPT-5.1 is OpenAI's frontier-grade model in the GPT-5 series, offering adaptive reasoning with configurable effort levels, improved coding and math performance, and a more natural conversational style compared to GPT-5.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2025-11-13", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-09-30", - resolvedAt: "2026-03-24T11:11:47.327Z", - baseModelName: "gpt-5.1", + "provider": "openai", + "description": "GPT-5.1 is OpenAI's frontier-grade model in the GPT-5 series, offering adaptive reasoning with configurable effort levels, improved coding and math performance, and a more natural conversational style compared to GPT-5.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2025-11-13", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-09-30", + "resolvedAt": "2026-03-24T11:11:47.327Z", + "baseModelName": "gpt-5.1" }, "gpt-5.2": { - provider: "openai", - description: - "OpenAI's flagship multimodal model released December 2025, excelling at long-context reasoning, agentic tool use, software engineering, and professional knowledge work. Available in Instant, Thinking, and Pro variants.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-12-11", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-08-31", - resolvedAt: "2026-03-24T11:11:13.129Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's flagship multimodal model released December 2025, excelling at long-context reasoning, agentic tool use, software engineering, and professional knowledge work. Available in Instant, Thinking, and Pro variants.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-12-11", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-08-31", + "resolvedAt": "2026-03-24T11:11:13.129Z", + "baseModelName": null }, "gpt-5.2-2025-12-11": { - provider: "openai", - description: - "OpenAI's flagship multimodal model released December 2025, excelling at long-context reasoning, agentic tool use, software engineering, and professional knowledge work. Available in Instant, Thinking, and Pro variants.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-12-11", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-08-31", - resolvedAt: "2026-03-24T11:11:13.129Z", - baseModelName: "gpt-5.2", + "provider": "openai", + "description": "OpenAI's flagship multimodal model released December 2025, excelling at long-context reasoning, agentic tool use, software engineering, and professional knowledge work. Available in Instant, Thinking, and Pro variants.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-12-11", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-08-31", + "resolvedAt": "2026-03-24T11:11:13.129Z", + "baseModelName": "gpt-5.2" }, "gpt-5.2-pro": { - provider: "openai", - description: - "OpenAI's previous pro-tier reasoning model optimized for complex professional work requiring step-by-step reasoning, instruction following, and accuracy in high-stakes use cases. Superseded by GPT-5.4 pro.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "extended_thinking"], - releaseDate: "2025-12-11", - isHidden: false, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-08-31", - resolvedAt: "2026-03-24T11:11:12.711Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's previous pro-tier reasoning model optimized for complex professional work requiring step-by-step reasoning, instruction following, and accuracy in high-stakes use cases. Superseded by GPT-5.4 pro.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "extended_thinking" + ], + "releaseDate": "2025-12-11", + "isHidden": false, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-08-31", + "resolvedAt": "2026-03-24T11:11:12.711Z", + "baseModelName": null }, "gpt-5.2-pro-2025-12-11": { - provider: "openai", - description: - "OpenAI's previous pro-tier reasoning model optimized for complex professional work requiring step-by-step reasoning, instruction following, and accuracy in high-stakes use cases. Superseded by GPT-5.4 pro.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "extended_thinking"], - releaseDate: "2025-12-11", - isHidden: false, - supportsStructuredOutput: false, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-08-31", - resolvedAt: "2026-03-24T11:11:12.711Z", - baseModelName: "gpt-5.2-pro", + "provider": "openai", + "description": "OpenAI's previous pro-tier reasoning model optimized for complex professional work requiring step-by-step reasoning, instruction following, and accuracy in high-stakes use cases. Superseded by GPT-5.4 pro.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "extended_thinking" + ], + "releaseDate": "2025-12-11", + "isHidden": false, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-08-31", + "resolvedAt": "2026-03-24T11:11:12.711Z", + "baseModelName": "gpt-5.2-pro" }, "gpt-5.4": { - provider: "openai", - description: - "OpenAI's most capable frontier model as of March 2026, featuring state-of-the-art coding, native computer-use capabilities, and a 1M-token context window for professional and agentic workflows.", - contextWindow: 1050000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "code_execution"], - releaseDate: "2026-03-05", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-08-31", - resolvedAt: "2026-03-24T11:12:09.220Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's most capable frontier model as of March 2026, featuring state-of-the-art coding, native computer-use capabilities, and a 1M-token context window for professional and agentic workflows.", + "contextWindow": 1050000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "code_execution" + ], + "releaseDate": "2026-03-05", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-08-31", + "resolvedAt": "2026-03-24T11:12:09.220Z", + "baseModelName": null }, "gpt-5.4-2026-03-05": { - provider: "openai", - description: - "OpenAI's most capable frontier model as of March 2026, featuring state-of-the-art coding, native computer-use capabilities, and a 1M-token context window for professional and agentic workflows.", - contextWindow: 1050000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "code_execution"], - releaseDate: "2026-03-05", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-08-31", - resolvedAt: "2026-03-24T11:12:09.220Z", - baseModelName: "gpt-5.4", + "provider": "openai", + "description": "OpenAI's most capable frontier model as of March 2026, featuring state-of-the-art coding, native computer-use capabilities, and a 1M-token context window for professional and agentic workflows.", + "contextWindow": 1050000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "code_execution" + ], + "releaseDate": "2026-03-05", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-08-31", + "resolvedAt": "2026-03-24T11:12:09.220Z", + "baseModelName": "gpt-5.4" }, "gpt-5.4-mini": { - provider: "openai", - description: - "OpenAI's fast and efficient small model from the GPT-5.4 family, designed for high-volume workloads. Approaches GPT-5.4 performance on coding and reasoning while running over 2x faster than GPT-5 mini.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2026-03-17", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-08-31", - resolvedAt: "2026-03-24T11:12:35.473Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's fast and efficient small model from the GPT-5.4 family, designed for high-volume workloads. Approaches GPT-5.4 performance on coding and reasoning while running over 2x faster than GPT-5 mini.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2026-03-17", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-08-31", + "resolvedAt": "2026-03-24T11:12:35.473Z", + "baseModelName": null }, "gpt-5.4-mini-2026-03-17": { - provider: "openai", - description: - "OpenAI's fast and efficient small model from the GPT-5.4 family, designed for high-volume workloads. Approaches GPT-5.4 performance on coding and reasoning while running over 2x faster than GPT-5 mini.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2026-03-17", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-08-31", - resolvedAt: "2026-03-24T11:12:35.473Z", - baseModelName: "gpt-5.4-mini", + "provider": "openai", + "description": "OpenAI's fast and efficient small model from the GPT-5.4 family, designed for high-volume workloads. Approaches GPT-5.4 performance on coding and reasoning while running over 2x faster than GPT-5 mini.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2026-03-17", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-08-31", + "resolvedAt": "2026-03-24T11:12:35.473Z", + "baseModelName": "gpt-5.4-mini" }, "gpt-5.4-nano": { - provider: "openai", - description: - "OpenAI's cheapest GPT-5.4-class model optimized for simple high-volume tasks like classification, data extraction, ranking, and sub-agent delegation in agentic workflows.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2026-03-17", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-08-31", - resolvedAt: "2026-03-24T11:12:52.285Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's cheapest GPT-5.4-class model optimized for simple high-volume tasks like classification, data extraction, ranking, and sub-agent delegation in agentic workflows.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2026-03-17", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-08-31", + "resolvedAt": "2026-03-24T11:12:52.285Z", + "baseModelName": null }, "gpt-5.4-nano-2026-03-17": { - provider: "openai", - description: - "OpenAI's cheapest GPT-5.4-class model optimized for simple high-volume tasks like classification, data extraction, ranking, and sub-agent delegation in agentic workflows.", - contextWindow: 400000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2026-03-17", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-08-31", - resolvedAt: "2026-03-24T11:12:52.285Z", - baseModelName: "gpt-5.4-nano", + "provider": "openai", + "description": "OpenAI's cheapest GPT-5.4-class model optimized for simple high-volume tasks like classification, data extraction, ranking, and sub-agent delegation in agentic workflows.", + "contextWindow": 400000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2026-03-17", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-08-31", + "resolvedAt": "2026-03-24T11:12:52.285Z", + "baseModelName": "gpt-5.4-nano" }, "gpt-5.4-pro": { - provider: "openai", - description: - "OpenAI's highest-capability GPT-5.4 variant, using additional compute for harder problems. Available via Responses API only, designed for complex reasoning, coding, and agentic workflows.", - contextWindow: 1050000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "extended_thinking"], - releaseDate: "2026-03-05", - isHidden: false, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: null, - knowledgeCutoff: "2025-08-31", - resolvedAt: "2026-03-24T11:12:56.903Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's highest-capability GPT-5.4 variant, using additional compute for harder problems. Available via Responses API only, designed for complex reasoning, coding, and agentic workflows.", + "contextWindow": 1050000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "extended_thinking" + ], + "releaseDate": "2026-03-05", + "isHidden": false, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": null, + "knowledgeCutoff": "2025-08-31", + "resolvedAt": "2026-03-24T11:12:56.903Z", + "baseModelName": null }, "gpt-5.4-pro-2026-03-05": { - provider: "openai", - description: - "OpenAI's highest-capability GPT-5.4 variant, using additional compute for harder problems. Available via Responses API only, designed for complex reasoning, coding, and agentic workflows.", - contextWindow: 1050000, - maxOutputTokens: 128000, - capabilities: ["vision", "tool_use", "streaming", "extended_thinking"], - releaseDate: "2026-03-05", - isHidden: false, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: null, - knowledgeCutoff: "2025-08-31", - resolvedAt: "2026-03-24T11:12:56.903Z", - baseModelName: "gpt-5.4-pro", - }, - o1: { - provider: "openai", - description: - "OpenAI's reasoning model designed for complex tasks requiring multi-step logical thinking, excelling at math, science, and coding problems.", - contextWindow: 200000, - maxOutputTokens: 100000, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2024-12-17", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:12:23.948Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's highest-capability GPT-5.4 variant, using additional compute for harder problems. Available via Responses API only, designed for complex reasoning, coding, and agentic workflows.", + "contextWindow": 1050000, + "maxOutputTokens": 128000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "extended_thinking" + ], + "releaseDate": "2026-03-05", + "isHidden": false, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": null, + "knowledgeCutoff": "2025-08-31", + "resolvedAt": "2026-03-24T11:12:56.903Z", + "baseModelName": "gpt-5.4-pro" + }, + "o1": { + "provider": "openai", + "description": "OpenAI's reasoning model designed for complex tasks requiring multi-step logical thinking, excelling at math, science, and coding problems.", + "contextWindow": 200000, + "maxOutputTokens": 100000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2024-12-17", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:12:23.948Z", + "baseModelName": null }, "o1-2024-12-17": { - provider: "openai", - description: - "OpenAI's reasoning model designed for complex tasks requiring multi-step logical thinking, excelling at math, science, and coding problems.", - contextWindow: 200000, - maxOutputTokens: 100000, - capabilities: ["vision", "tool_use", "streaming", "json_mode"], - releaseDate: "2024-12-17", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:12:23.948Z", - baseModelName: "o1", + "provider": "openai", + "description": "OpenAI's reasoning model designed for complex tasks requiring multi-step logical thinking, excelling at math, science, and coding problems.", + "contextWindow": 200000, + "maxOutputTokens": 100000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode" + ], + "releaseDate": "2024-12-17", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:12:23.948Z", + "baseModelName": "o1" }, "o1-mini": { - provider: "openai", - description: - "A smaller, faster, and cheaper reasoning model in OpenAI's o1 series, optimized for coding, math, and science tasks requiring multi-step reasoning.", - contextWindow: 128000, - maxOutputTokens: 65536, - capabilities: ["streaming", "json_mode"], - releaseDate: "2024-09-12", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: "2025-06-30", - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:12:37.030Z", - baseModelName: null, + "provider": "openai", + "description": "A smaller, faster, and cheaper reasoning model in OpenAI's o1 series, optimized for coding, math, and science tasks requiring multi-step reasoning.", + "contextWindow": 128000, + "maxOutputTokens": 65536, + "capabilities": [ + "streaming", + "json_mode" + ], + "releaseDate": "2024-09-12", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": "2025-06-30", + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:12:37.030Z", + "baseModelName": null }, "o1-mini-2024-09-12": { - provider: "openai", - description: - "A smaller, faster, and cheaper reasoning model in OpenAI's o1 series, optimized for coding, math, and science tasks requiring multi-step reasoning.", - contextWindow: 128000, - maxOutputTokens: 65536, - capabilities: ["streaming", "json_mode"], - releaseDate: "2024-09-12", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: "2025-06-30", - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:12:37.030Z", - baseModelName: "o1-mini", + "provider": "openai", + "description": "A smaller, faster, and cheaper reasoning model in OpenAI's o1 series, optimized for coding, math, and science tasks requiring multi-step reasoning.", + "contextWindow": 128000, + "maxOutputTokens": 65536, + "capabilities": [ + "streaming", + "json_mode" + ], + "releaseDate": "2024-09-12", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": "2025-06-30", + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:12:37.030Z", + "baseModelName": "o1-mini" }, "o1-preview": { - provider: "openai", - description: - "OpenAI's first reasoning model using chain-of-thought to solve complex problems in science, coding, and math. Predecessor to o1 and o3 series.", - contextWindow: 128000, - maxOutputTokens: 32768, - capabilities: ["streaming"], - releaseDate: "2024-09-12", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: "2025-10-31", - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:12:59.198Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's first reasoning model using chain-of-thought to solve complex problems in science, coding, and math. Predecessor to o1 and o3 series.", + "contextWindow": 128000, + "maxOutputTokens": 32768, + "capabilities": [ + "streaming" + ], + "releaseDate": "2024-09-12", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": "2025-10-31", + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:12:59.198Z", + "baseModelName": null }, "o1-preview-2024-09-12": { - provider: "openai", - description: - "OpenAI's first reasoning model using chain-of-thought to solve complex problems in science, coding, and math. Predecessor to o1 and o3 series.", - contextWindow: 128000, - maxOutputTokens: 32768, - capabilities: ["streaming"], - releaseDate: "2024-09-12", - isHidden: true, - supportsStructuredOutput: false, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: "2025-10-31", - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:12:59.198Z", - baseModelName: "o1-preview", + "provider": "openai", + "description": "OpenAI's first reasoning model using chain-of-thought to solve complex problems in science, coding, and math. Predecessor to o1 and o3 series.", + "contextWindow": 128000, + "maxOutputTokens": 32768, + "capabilities": [ + "streaming" + ], + "releaseDate": "2024-09-12", + "isHidden": true, + "supportsStructuredOutput": false, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": "2025-10-31", + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:12:59.198Z", + "baseModelName": "o1-preview" }, "o1-pro": { - provider: "openai", - description: - "A version of OpenAI's o1 reasoning model that uses significantly more compute to deliver better, more consistent answers on complex reasoning tasks in science, coding, and math.", - contextWindow: 200000, - maxOutputTokens: 100000, - capabilities: ["vision", "tool_use", "json_mode", "extended_thinking"], - releaseDate: "2025-03-19", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: null, - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:13:57.532Z", - baseModelName: null, + "provider": "openai", + "description": "A version of OpenAI's o1 reasoning model that uses significantly more compute to deliver better, more consistent answers on complex reasoning tasks in science, coding, and math.", + "contextWindow": 200000, + "maxOutputTokens": 100000, + "capabilities": [ + "vision", + "tool_use", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-03-19", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": null, + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:13:57.532Z", + "baseModelName": null }, "o1-pro-2025-03-19": { - provider: "openai", - description: - "A version of OpenAI's o1 reasoning model that uses significantly more compute to deliver better, more consistent answers on complex reasoning tasks in science, coding, and math.", - contextWindow: 200000, - maxOutputTokens: 100000, - capabilities: ["vision", "tool_use", "json_mode", "extended_thinking"], - releaseDate: "2025-03-19", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: null, - knowledgeCutoff: "2023-10-01", - resolvedAt: "2026-03-24T11:13:57.532Z", - baseModelName: "o1-pro", - }, - o3: { - provider: "openai", - description: - "OpenAI's advanced reasoning model designed for complex tasks requiring deep reasoning, excelling at software engineering, mathematics, scientific reasoning, and visual reasoning tasks.", - contextWindow: 200000, - maxOutputTokens: 100000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-04-16", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-06-01", - resolvedAt: "2026-03-24T11:14:04.906Z", - baseModelName: null, + "provider": "openai", + "description": "A version of OpenAI's o1 reasoning model that uses significantly more compute to deliver better, more consistent answers on complex reasoning tasks in science, coding, and math.", + "contextWindow": 200000, + "maxOutputTokens": 100000, + "capabilities": [ + "vision", + "tool_use", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-03-19", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": null, + "knowledgeCutoff": "2023-10-01", + "resolvedAt": "2026-03-24T11:13:57.532Z", + "baseModelName": "o1-pro" + }, + "o3": { + "provider": "openai", + "description": "OpenAI's advanced reasoning model designed for complex tasks requiring deep reasoning, excelling at software engineering, mathematics, scientific reasoning, and visual reasoning tasks.", + "contextWindow": 200000, + "maxOutputTokens": 100000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-04-16", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-06-01", + "resolvedAt": "2026-03-24T11:14:04.906Z", + "baseModelName": null }, "o3-2025-04-16": { - provider: "openai", - description: - "OpenAI's advanced reasoning model designed for complex tasks requiring deep reasoning, excelling at software engineering, mathematics, scientific reasoning, and visual reasoning tasks.", - contextWindow: 200000, - maxOutputTokens: 100000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-04-16", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-06-01", - resolvedAt: "2026-03-24T11:14:04.906Z", - baseModelName: "o3", + "provider": "openai", + "description": "OpenAI's advanced reasoning model designed for complex tasks requiring deep reasoning, excelling at software engineering, mathematics, scientific reasoning, and visual reasoning tasks.", + "contextWindow": 200000, + "maxOutputTokens": 100000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-04-16", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-06-01", + "resolvedAt": "2026-03-24T11:14:04.906Z", + "baseModelName": "o3" }, "o3-mini": { - provider: "openai", - description: - "OpenAI's compact reasoning model optimized for STEM tasks, offering strong performance in math, science, and coding at lower cost than o3.", - contextWindow: 200000, - maxOutputTokens: 100000, - capabilities: ["tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-01-31", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-01-01", - resolvedAt: "2026-03-24T11:13:33.788Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's compact reasoning model optimized for STEM tasks, offering strong performance in math, science, and coding at lower cost than o3.", + "contextWindow": 200000, + "maxOutputTokens": 100000, + "capabilities": [ + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-01-31", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-01-01", + "resolvedAt": "2026-03-24T11:13:33.788Z", + "baseModelName": null }, "o3-mini-2025-01-31": { - provider: "openai", - description: - "OpenAI's compact reasoning model optimized for STEM tasks, offering strong performance in math, science, and coding at lower cost than o3.", - contextWindow: 200000, - maxOutputTokens: 100000, - capabilities: ["tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-01-31", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2025-01-01", - resolvedAt: "2026-03-24T11:13:33.788Z", - baseModelName: "o3-mini", + "provider": "openai", + "description": "OpenAI's compact reasoning model optimized for STEM tasks, offering strong performance in math, science, and coding at lower cost than o3.", + "contextWindow": 200000, + "maxOutputTokens": 100000, + "capabilities": [ + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-01-31", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2025-01-01", + "resolvedAt": "2026-03-24T11:13:33.788Z", + "baseModelName": "o3-mini" }, "o3-pro": { - provider: "openai", - description: - "OpenAI's most reliable reasoning model, a version of o3 designed to think longer and provide more consistently accurate answers for challenging math, science, and coding problems.", - contextWindow: 200000, - maxOutputTokens: 100000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-06-10", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: null, - knowledgeCutoff: "2024-06-01", - resolvedAt: "2026-03-24T11:14:10.900Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's most reliable reasoning model, a version of o3 designed to think longer and provide more consistently accurate answers for challenging math, science, and coding problems.", + "contextWindow": 200000, + "maxOutputTokens": 100000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-06-10", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": null, + "knowledgeCutoff": "2024-06-01", + "resolvedAt": "2026-03-24T11:14:10.900Z", + "baseModelName": null }, "o3-pro-2025-06-10": { - provider: "openai", - description: - "OpenAI's most reliable reasoning model, a version of o3 designed to think longer and provide more consistently accurate answers for challenging math, science, and coding problems.", - contextWindow: 200000, - maxOutputTokens: 100000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-06-10", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: false, - supportsStreamingToolCalls: false, - deprecationDate: null, - knowledgeCutoff: "2024-06-01", - resolvedAt: "2026-03-24T11:14:10.900Z", - baseModelName: "o3-pro", + "provider": "openai", + "description": "OpenAI's most reliable reasoning model, a version of o3 designed to think longer and provide more consistently accurate answers for challenging math, science, and coding problems.", + "contextWindow": 200000, + "maxOutputTokens": 100000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-06-10", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": false, + "supportsStreamingToolCalls": false, + "deprecationDate": null, + "knowledgeCutoff": "2024-06-01", + "resolvedAt": "2026-03-24T11:14:10.900Z", + "baseModelName": "o3-pro" }, "o4-mini": { - provider: "openai", - description: - "OpenAI's small reasoning model optimized for fast, cost-efficient reasoning with strong performance in math, coding, and visual tasks.", - contextWindow: 200000, - maxOutputTokens: 100000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-04-16", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-06-01", - resolvedAt: "2026-03-24T11:14:16.050Z", - baseModelName: null, + "provider": "openai", + "description": "OpenAI's small reasoning model optimized for fast, cost-efficient reasoning with strong performance in math, coding, and visual tasks.", + "contextWindow": 200000, + "maxOutputTokens": 100000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-04-16", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-06-01", + "resolvedAt": "2026-03-24T11:14:16.050Z", + "baseModelName": null }, "o4-mini-2025-04-16": { - provider: "openai", - description: - "OpenAI's small reasoning model optimized for fast, cost-efficient reasoning with strong performance in math, coding, and visual tasks.", - contextWindow: 200000, - maxOutputTokens: 100000, - capabilities: ["vision", "tool_use", "streaming", "json_mode", "extended_thinking"], - releaseDate: "2025-04-16", - isHidden: false, - supportsStructuredOutput: true, - supportsParallelToolCalls: true, - supportsStreamingToolCalls: true, - deprecationDate: null, - knowledgeCutoff: "2024-06-01", - resolvedAt: "2026-03-24T11:14:16.050Z", - baseModelName: "o4-mini", - }, + "provider": "openai", + "description": "OpenAI's small reasoning model optimized for fast, cost-efficient reasoning with strong performance in math, coding, and visual tasks.", + "contextWindow": 200000, + "maxOutputTokens": 100000, + "capabilities": [ + "vision", + "tool_use", + "streaming", + "json_mode", + "extended_thinking" + ], + "releaseDate": "2025-04-16", + "isHidden": false, + "supportsStructuredOutput": true, + "supportsParallelToolCalls": true, + "supportsStreamingToolCalls": true, + "deprecationDate": null, + "knowledgeCutoff": "2024-06-01", + "resolvedAt": "2026-03-24T11:14:16.050Z", + "baseModelName": "o4-mini" + } };