diff --git a/docs/docs/development/type-generation.md b/docs/docs/development/type-generation.md index 17646e290..506a7a81b 100644 --- a/docs/docs/development/type-generation.md +++ b/docs/docs/development/type-generation.md @@ -93,6 +93,24 @@ Pass `--wait` for CI and production builds, where accurate types must be present npx @databricks/appkit generate-types --wait ``` +#### Wait limit + +`--wait` never waits forever. If the warehouse hasn't reached `RUNNING` within the limit (default **300 seconds**), the generator stops waiting: + +- If committed types exist, it keeps them, prints a warning and exits 0. The warning includes the error code, the warehouse ID, the state it was stuck in and the limit, e.g. `warehouse_unavailable: warehouse abc123 is still STARTING after the 300s wait limit; kept existing analytics.d.ts`. +- If there are no committed types to fall back on, it fails with exit code 1 and a message with the same `warehouse_unavailable: …` line. + +One limit covers the whole run: query types and metric-view types share it, so a stuck warehouse costs at most one limit. A warehouse that is already `RUNNING` is not affected. + +The limit also holds when the warehouse API stops answering: every status request made while waiting is cut off when the limit runs out. The first status check, and the start request for a stopped warehouse, each get the time left in the limit, but at least 10 seconds and at most 30 seconds. So with a limit under 10 seconds, a slow API can still take up to 10 seconds to answer. + +Set the limit in seconds with the `--wait-timeout` flag or the `APPKIT_TYPEGEN_WAIT_TIMEOUT` environment variable (the flag wins). `0` checks the warehouse once and doesn't wait. The environment variable also applies to `appKitTypesPlugin()` when it runs in a production Vite build. + +```bash +npx @databricks/appkit generate-types --wait --wait-timeout 120 +APPKIT_TYPEGEN_WAIT_TIMEOUT=120 npm run build +``` + #### CI resilience: committed types as fallback In blocking mode (`--wait`), the generator attempts to fetch real types from your warehouse, but delegates to **committed type files** (`shared/appkit-types/analytics.d.ts` and, when Metric Views are configured, `shared/appkit-types/metric-views.d.ts`) as the fallback when the warehouse is unreachable. These generated files should be part of your repository. On a fresh CI checkout, every build attempts to DESCRIBE against the warehouse; the committed types are used only when that cannot complete. @@ -102,7 +120,7 @@ The generator **never overwrites committed types with degraded (`result: unknown A **two-bucket failure taxonomy** determines whether the build crashes or falls back to committed types: - **Deterministic failures (always crash):** SQL syntax errors in your queries (genuine DESCRIBE failure against a reachable warehouse), HTTP 404 (bad or unknown warehouse ID), HTTP 400 (malformed request). These are developer or configuration errors that committed types must not hide. -- **Environmental failures (gate on committed types):** Authentication failures (401/403), network unreachability, warehouse unavailability (cold, deleting, or deleted), wait timeout on `RUNNING`, or any unrecognized failure. If every type file required by the app exists, the build **keeps them, emits a loud warning to stderr, and succeeds (exit 0)**. If a required file is missing, the build **crashes** with a message instructing you to run `npx @databricks/appkit generate-types --wait` locally (against a reachable warehouse) and commit the generated type files. +- **Environmental failures (gate on committed types):** Authentication failures (401/403), network unreachability, warehouse unavailability (cold, deleting, or deleted), the [wait limit](#wait-limit) running out before `RUNNING`, or any unrecognized failure. If every type file required by the app exists, the build **keeps them, emits a loud warning to stderr, and succeeds (exit 0)**. If a required file is missing, the build **crashes** with a message instructing you to run `npx @databricks/appkit generate-types --wait` locally (against a reachable warehouse) and commit the generated type files. The loud warning is a single greppable stderr line naming the coarse cause (auth blocked / warehouse unreachable / warehouse unavailable) and the warehouse ID, so CI logs surface that the build fell back to committed types. diff --git a/packages/appkit/src/type-generator/index.ts b/packages/appkit/src/type-generator/index.ts index e4ea97fd3..d5d79887c 100644 --- a/packages/appkit/src/type-generator/index.ts +++ b/packages/appkit/src/type-generator/index.ts @@ -47,9 +47,18 @@ import { generateServingTypes as generateServingTypesImpl } from "./serving/gene import { createTypegenWorkspaceClient } from "./typegen-client"; import type { QueryFatalError, QuerySchema, QuerySyntaxError } from "./types"; import { + describeStatusTimeout, + describeWaitTimeout, getWarehouseState, + remainingWaitMs, + resolveWarehouseWaitTimeoutMs, startWarehouse, + statusRequestTimeoutMs, + WAREHOUSE_UNAVAILABLE_CODE, + WAREHOUSE_WAIT_TIMEOUT_ENV, type WarehouseState, + WarehouseStatusTimeoutError, + WarehouseWaitTimeoutError, waitUntilRunning, } from "./warehouse-status"; @@ -57,16 +66,11 @@ dotenv.config(); const logger = createLogger("type-generator"); -/** - * Upper bound (~5 min) on how long the Metric Views path's `blocking`-mode preflight - * waits for a warehouse to reach RUNNING. Mirrors the query path's (unexported) - * `PREFLIGHT_WAIT_MAX_MS` in query-registry.ts. - */ -const MV_PREFLIGHT_WAIT_MAX_MS = 300_000; - function determineWarningMessage( cause: "auth" | "unreachable" | "unavailable", warehouseId: string, + detail?: string, + keptFiles: string[] = [], ): string { const causeLabel = cause === "auth" @@ -75,7 +79,11 @@ function determineWarningMessage( ? "warehouse unreachable" : "warehouse unavailable"; // Use a stable prefix for greppability and CI log matching. - return `AppKit typegen: using committed types — warehouse ${warehouseId} ${causeLabel}; please check warehouse status and retry`; + const base = `AppKit typegen: using committed types — warehouse ${warehouseId} ${causeLabel}; please check warehouse status and retry`; + if (!detail) return base; + const kept = + keptFiles.length > 0 ? `; kept existing ${keptFiles.join(", ")}` : ""; + return `${base} (${detail}${kept})`; } type TypegenFailure = QuerySyntaxError | QueryFatalError; @@ -136,6 +144,8 @@ function formatTypegenFailureMessage(options: { title: string; causes: string[]; nextStep: string; + /** Replaces the default "N queries could not be described" summary lines. */ + summary?: string[]; }) { const { syntaxErrors, fatalErrors = [], warehouseId, title } = options; const total = syntaxErrors.length + fatalErrors.length; @@ -143,12 +153,15 @@ function formatTypegenFailureMessage(options: { const warehouse = warehouseId ? ` against ${pc.dim(`warehouse ${warehouseId}`)}` : ""; + const summary = options.summary ?? [ + `${title}: ${total} ${plural(total, "query", "queries")} could not be described${warehouse}.`, + `AppKit wrote generated types with ${pc.bold("result: unknown")} for the failed ${plural(total, "query", "queries")}.`, + ]; return [ ` ${pc.bold(pc.red("Type generation failed"))}`, ` ${separator}`, - ` ${title}: ${total} ${plural(total, "query", "queries")} could not be described${warehouse}.`, - ` AppKit wrote generated types with ${pc.bold("result: unknown")} for the failed ${plural(total, "query", "queries")}.`, + ...summary.map((line) => ` ${line}`), "", ...formatFailureRows("SQL ERR", syntaxErrors, pc.red), ...(syntaxErrors.length > 0 && fatalErrors.length > 0 ? [""] : []), @@ -205,33 +218,84 @@ export class TypegenSyntaxError extends Error { * setup/request problem, such as missing permissions, invalid warehouse IDs, or * malformed SDK configuration. Like TypegenSyntaxError, this is thrown only * after the declaration file has been written with `result: unknown` entries. + * + * Also thrown, with `code: "warehouse_unavailable"`, when `--wait` ran out of + * time waiting for the warehouse and there are no committed types to keep (see + * {@link warehouseUnavailableError}). */ export class TypegenFatalError extends Error { readonly queries: QueryFatalError[]; + /** Machine-readable reason, when there is one (e.g. `warehouse_unavailable`). */ + readonly code?: string; - constructor(queries: QueryFatalError[], warehouseId?: string) { + constructor( + queries: QueryFatalError[], + warehouseId?: string, + format?: { + code?: string; + summary: string[]; + causes: string[]; + nextStep: string; + }, + ) { super( formatTypegenFailureMessage({ syntaxErrors: [], fatalErrors: queries, warehouseId, title: "DESCRIBE QUERY could not be requested", - causes: [ + summary: format?.summary, + causes: format?.causes ?? [ "missing warehouse permissions", "invalid warehouse ID", "authentication failure", "SDK configuration errors", ], - nextStep: warehouseId - ? `Verify access to warehouse ${pc.bold(warehouseId)} and rerun type generation.` - : "Verify warehouse access and rerun type generation.", + nextStep: + format?.nextStep ?? + (warehouseId + ? `Verify access to warehouse ${pc.bold(warehouseId)} and rerun type generation.` + : "Verify warehouse access and rerun type generation."), }), ); this.name = "TypegenFatalError"; this.queries = queries; + this.code = format?.code; } } +/** + * The `--wait` failure for a warehouse that didn't reach RUNNING within the + * wait limit when no committed types exist to fall back on. + */ +function warehouseUnavailableError( + warehouseId: string, + detail: string, + missingFiles: string[], +): TypegenFatalError { + return new TypegenFatalError( + [ + { + name: "type-generator", + message: `${detail}; no committed types to fall back on (missing: ${missingFiles.join(", ")})`, + }, + ], + warehouseId, + { + code: WAREHOUSE_UNAVAILABLE_CODE, + summary: [ + `Warehouse ${pc.bold(warehouseId)} was not ready in time, and there are no committed types to use instead.`, + "No type files were written.", + ], + causes: [ + "the warehouse is still starting (slow cold start, capacity, or a stuck start)", + "the warehouse was stopped and did not finish starting within the limit", + ], + nextStep: `Check warehouse ${pc.bold(warehouseId)} in the Databricks UI and rerun. If it just needs longer, raise the limit with --wait-timeout or ${WAREHOUSE_WAIT_TIMEOUT_ENV}. To build without the warehouse, run 'npx @databricks/appkit generate-types --wait' where it is reachable and commit the generated type files.`, + }, + ); +} + /** * Generate type declarations for QueryRegistry * Create the d.ts file from the plugin routes and query schemas @@ -304,6 +368,12 @@ async function probeWarehouseState( * only when the warehouse is already RUNNING, otherwise emits permissive * degraded types immediately. `"blocking"` waits for / starts the warehouse * first, failing the build only for a deleted/deleting one. + * @param options.waitTimeoutMs - `"blocking"` only: the limit, in milliseconds, + * on waiting for the warehouse to reach RUNNING. One budget covers the whole + * run (queries and metric views). Defaults to `APPKIT_TYPEGEN_WAIT_TIMEOUT` + * (seconds), else 5 minutes. When it runs out, committed types are kept with + * a warning, or — when there are none — the run throws a + * {@link TypegenFatalError} naming the warehouse, its state and the limit. * @param options.metricViewsFolder - folder that holds `definitions.json` * (`/config/metric-views`). Optional and independent of `queryFolder`: * metric-view types generate whenever this folder holds a config, even if the @@ -326,6 +396,7 @@ export async function generateFromEntryPoint(options: { warehouseId: string; noCache?: boolean; mode?: PreflightMode; + waitTimeoutMs?: number; mvOutFile?: string; metricFetcher?: DescribeFetcher; }) { @@ -351,6 +422,19 @@ export async function generateFromEntryPoint(options: { const projectRoot = resolveProjectRoot(outFile); + // One wait budget for the whole run: the metric-view preflight gets whatever + // the query preflight left, so a stuck warehouse costs at most one limit. + const waitTimeoutMs = resolveWarehouseWaitTimeoutMs( + options.waitTimeoutMs, + (raw) => + logger.warn( + "Ignoring %s=%s: expected a number of seconds (>= 0). Using the default wait limit.", + WAREHOUSE_WAIT_TIMEOUT_ENV, + raw, + ), + ); + const waitDeadline = Date.now() + waitTimeoutMs; + logger.debug("Starting type generation..."); let queryRegistry: QuerySchema[] = []; @@ -361,11 +445,15 @@ export async function generateFromEntryPoint(options: { let metricsHadEnvironmentalFailure = false; // Track the coarse cause of the environmental failure for the warning message. let environmentalCause: "auth" | "unreachable" | "unavailable" | undefined; + // A more specific reason, when known (e.g. the wait limit ran out). + let environmentalDetail: string | undefined; if (queryFolder) { const result = await generateQueriesFromDescribe(queryFolder, warehouseId, { noCache, mode, + waitTimeoutMs, + waitDeadline, }); queryRegistry = result.schemas; syntaxErrors = result.syntaxErrors ?? []; @@ -373,6 +461,7 @@ export async function generateFromEntryPoint(options: { queryHadEnvironmentalFailure = result.hadEnvironmentalFailure ?? false; environmentalCause = environmentalCause ?? result.environmentalCause ?? undefined; + environmentalDetail = result.environmentalDetail; } const typeDeclarations = generateTypeDeclarations(queryRegistry); @@ -409,6 +498,8 @@ export async function generateFromEntryPoint(options: { cache: !noCache, metricFetcher, mode, + waitTimeoutMs, + waitDeadline, // In blocking mode, never overwrite committed metric types with a // degraded result — including on runs that go on to throw, so a failing // build leaves the committed type file intact. Non-blocking always writes. @@ -439,6 +530,7 @@ export async function generateFromEntryPoint(options: { (mode === "blocking" && hasAnyDegradedMetrics(mvResult.schemas)); environmentalCause = environmentalCause ?? mvResult.environmentalCause ?? undefined; + environmentalDetail = environmentalDetail ?? mvResult.environmentalDetail; // Blocking (`--wait` / prod Vite) escalates only deterministic per-key // DESCRIBE failures. Transient connectivity failures are already recorded @@ -480,12 +572,26 @@ export async function generateFromEntryPoint(options: { if (missingCommittedTypes.length === 0) { // Committed types present: emit loud warning and exit 0. + const keptFiles: string[] = []; + if (queryHadEnvironmentalFailure) keptFiles.push(path.basename(outFile)); + if (metricsHadEnvironmentalFailure) { + keptFiles.push(path.basename(resolvedMvFile)); + } const warningMessage = determineWarningMessage( environmentalCause ?? "unavailable", warehouseId, + environmentalDetail, + keptFiles, ); logger.warn(warningMessage); } else { + if (environmentalDetail) { + throw warehouseUnavailableError( + warehouseId, + environmentalDetail, + missingCommittedTypes, + ); + } throw new TypegenFatalError( [ { @@ -539,6 +645,8 @@ export interface SyncMetricViewsTypesResult { * hadEnvironmentalFailure is true; used by the warning message. */ environmentalCause?: "auth" | "unreachable" | "unavailable"; + /** Specific reason for the environmental failure (e.g. wait limit reached). */ + environmentalDetail?: string; } /** @@ -557,6 +665,8 @@ export interface SyncMetricViewsTypesResult { * @param options.metricFetcher - optional injected {@link DescribeFetcher} * @param options.mode - preflight/gate policy, default `"describe-now"`. When set to `"blocking"`, * metric-view `.ts` writes are suppressed if any metric is degraded (to preserve committed files). + * @param options.waitTimeoutMs - `"blocking"` only: limit on waiting for the + * warehouse to reach RUNNING (see {@link generateFromEntryPoint}). * @param options.suppressDegradedWrite - when true (only in `mode === "blocking"` context), skip * the metricOutFile write if any metric schema has `degraded === true`. Used to prevent * overwriting committed type files with degraded types in blocking mode. @@ -568,6 +678,9 @@ export async function syncMetricViewsTypes(options: { cache?: boolean; metricFetcher?: DescribeFetcher; mode?: "describe-now" | "non-blocking" | "blocking"; + waitTimeoutMs?: number; + /** Internal: epoch-ms deadline shared with the query wait. */ + waitDeadline?: number; suppressDegradedWrite?: boolean; }): Promise { const { @@ -582,6 +695,7 @@ export async function syncMetricViewsTypes(options: { // Only `cache === false` disables caching; `undefined`/`true` keep it on. const noCache = cacheEnabled === false; + const waitTimeoutMs = resolveWarehouseWaitTimeoutMs(options.waitTimeoutMs); const mvConfig = await readMetricConfig(metricViewsFolder); if (!mvConfig) { @@ -636,17 +750,22 @@ export async function syncMetricViewsTypes(options: { }; // Blocking-mode preflight: ensure the warehouse is running before the MV DESCRIBE - // batch (probe → decide → wait / start+wait; only DELETED/DELETING is fatal). Two softenings vs the query preflight: a failed probe and a timed-out wait are NOT fatal here — we fall through to syncMetrics, which classifies a still-not-ready warehouse as degraded rather than failing the build. Skipped for `describe-now`/`non-blocking` (only `mode === "blocking"` enters here). + // batch (probe → decide → wait / start+wait; only DELETED/DELETING is fatal). A connectivity blip falls through to syncMetrics, which classifies a still-not-ready warehouse as degraded; a timed-out wait skips the DESCRIBEs and degrades for the committed-types gate. Skipped for `describe-now`/`non-blocking` (only `mode === "blocking"` enters here). let preflightFatalMessage: string | undefined; let hadEnvironmentalFailure = false; let environmentalCause: "auth" | "unreachable" | "unavailable" | undefined; + let environmentalDetail: string | undefined; if ( mode === "blocking" && metricFetcher === undefined && describeNeeded.length > 0 ) { try { - const state = await getWarehouseState(getMvClient(), warehouseId); + const state = await getWarehouseState(getMvClient(), warehouseId, { + timeoutMs: statusRequestTimeoutMs( + remainingWaitMs(waitTimeoutMs, options.waitDeadline), + ), + }); const decision = decidePreflight(state, mode); if (decision === "fatal") { preflightFatalMessage = `warehouse ${warehouseId} is ${state}`; @@ -656,10 +775,15 @@ export async function syncMetricViewsTypes(options: { } else if (decision === "startWaitProceed") { // treatStoppedAsTransient rides out the stale pre-start STOPPED/STOPPING // reading, same as the query preflight. - await startWarehouse(getMvClient(), warehouseId); + await startWarehouse(getMvClient(), warehouseId, { + timeoutMs: statusRequestTimeoutMs( + remainingWaitMs(waitTimeoutMs, options.waitDeadline), + ), + }); const settled = await waitUntilRunning(getMvClient(), warehouseId, { - maxMs: MV_PREFLIGHT_WAIT_MAX_MS, + maxMs: remainingWaitMs(waitTimeoutMs, options.waitDeadline), treatStoppedAsTransient: true, + lastKnownState: state, }); if (settled !== "RUNNING") { // With treatStoppedAsTransient, a non-RUNNING resolve is exactly @@ -669,7 +793,8 @@ export async function syncMetricViewsTypes(options: { } } else if (decision === "waitThenProceed") { const settled = await waitUntilRunning(getMvClient(), warehouseId, { - maxMs: MV_PREFLIGHT_WAIT_MAX_MS, + maxMs: remainingWaitMs(waitTimeoutMs, options.waitDeadline), + lastKnownState: state, }); if (settled === "DELETED" || settled === "DELETING") { // Deleted mid-wait: fatal. Environmental (state-based). @@ -678,9 +803,22 @@ export async function syncMetricViewsTypes(options: { } } } catch (err) { - // Connectivity blip: fall through to syncMetrics, whose DESCRIBEs degrade - // a not-ready / unreachable warehouse rather than throwing. - if (!isConnectivityError(err)) { + if (err instanceof WarehouseWaitTimeoutError) { + // Wait limit ran out: skip the DESCRIBEs (they'd only queue on the + // still-starting warehouse) and degrade for the committed-types gate. + preflightFatalMessage = describeWaitTimeout(err, waitTimeoutMs); + hadEnvironmentalFailure = true; + environmentalCause = "unavailable"; + environmentalDetail = preflightFatalMessage; + } else if (err instanceof WarehouseStatusTimeoutError) { + // The warehouse API itself didn't answer in time: same handling. + preflightFatalMessage = describeStatusTimeout(err); + hadEnvironmentalFailure = true; + environmentalCause = "unavailable"; + environmentalDetail = preflightFatalMessage; + } else if (!isConnectivityError(err)) { + // Connectivity blip: fall through to syncMetrics, whose DESCRIBEs + // degrade a not-ready / unreachable warehouse rather than throwing. // Deterministic failures become fatal errors; environmental failures // degrade for the committed-types gate. const classification = classifyBlockingFailure(err); @@ -890,6 +1028,10 @@ export async function syncMetricViewsTypes(options: { mode === "blocking" && hadEnvironmentalFailure ? environmentalCause : undefined, + environmentalDetail: + mode === "blocking" && hadEnvironmentalFailure + ? environmentalDetail + : undefined, }; } diff --git a/packages/appkit/src/type-generator/query-registry.ts b/packages/appkit/src/type-generator/query-registry.ts index 8ccdb1ef0..cff25109b 100644 --- a/packages/appkit/src/type-generator/query-registry.ts +++ b/packages/appkit/src/type-generator/query-registry.ts @@ -31,20 +31,20 @@ import { sqlTypeToMarker, } from "./types"; import { + describeStatusTimeout, + describeWaitTimeout, getWarehouseState, + remainingWaitMs, + resolveWarehouseWaitTimeoutMs, startWarehouse, + statusRequestTimeoutMs, + WarehouseStatusTimeoutError, + WarehouseWaitTimeoutError, waitUntilRunning, } from "./warehouse-status"; const logger = createLogger("type-generator:query-registry"); -/** - * Upper bound on how long a `blocking`-mode preflight will wait for a starting - * warehouse to reach RUNNING before giving up (~5 min). Generous enough to ride - * out a cold start without hanging an interactive CLI invocation indefinitely. - */ -const PREFLIGHT_WAIT_MAX_MS = 300_000; - /** * Numeric-context patterns for positional type inference. * Hoisted to module scope — safe because matchAll() clones the regex internally. @@ -498,6 +498,11 @@ export function inferParameterTypes( * immediately), "blocking" waits for a starting warehouse and starts (then * waits for) a stopped one, treating only a deleted/deleting warehouse as * fatal. Defaults to "non-blocking". + * @param options.waitTimeoutMs - "blocking" only: how long to wait for the + * warehouse to reach RUNNING. Defaults to {@link resolveWarehouseWaitTimeoutMs} + * (`APPKIT_TYPEGEN_WAIT_TIMEOUT` seconds, else 5 min). When it runs out the + * run degrades like any environmental failure and reports the timeout in + * `environmentalDetail`. * @returns an array of query schemas */ export async function generateQueriesFromDescribe( @@ -507,6 +512,9 @@ export async function generateQueriesFromDescribe( noCache?: boolean; concurrency?: number; mode?: PreflightMode; + waitTimeoutMs?: number; + /** Internal: epoch-ms deadline shared with the metric-view wait. */ + waitDeadline?: number; } = {}, ): Promise { const { @@ -514,6 +522,7 @@ export async function generateQueriesFromDescribe( concurrency: rawConcurrency = 10, mode = "non-blocking", } = options; + const waitTimeoutMs = resolveWarehouseWaitTimeoutMs(options.waitTimeoutMs); const concurrency = typeof rawConcurrency === "number" && Number.isFinite(rawConcurrency) ? Math.max(1, Math.floor(rawConcurrency)) @@ -651,6 +660,9 @@ export async function generateQueriesFromDescribe( // gate's warning can say why generation fell back to committed types. let hadEnvironmentalFailure = false; let environmentalCause: "auth" | "unreachable" | "unavailable" | undefined; + // Human-readable reason for the environmental failure, when there is a more + // specific one than the coarse cause (e.g. the wait limit ran out). + let environmentalDetail: string | undefined; if (uncachedQueries.length > 0) { // One-time warehouse preflight (before issuing any DESCRIBE). A single @@ -672,7 +684,11 @@ export async function generateQueriesFromDescribe( decision = "degradeAll"; } else { try { - const state = await getWarehouseState(client, warehouseId); + const state = await getWarehouseState(client, warehouseId, { + timeoutMs: statusRequestTimeoutMs( + remainingWaitMs(waitTimeoutMs, options.waitDeadline), + ), + }); decision = decidePreflight(state, mode); if (decision === "fatal") { // DELETED/DELETING is state-based and environmental. @@ -685,10 +701,15 @@ export async function generateQueriesFromDescribe( // poll to RUNNING. treatStoppedAsTransient rides out the stale // pre-start STOPPED/STOPPING reading the start hasn't propagated past // yet — only DELETED/DELETING (or the deadline) ends the wait early. - await startWarehouse(client, warehouseId); + await startWarehouse(client, warehouseId, { + timeoutMs: statusRequestTimeoutMs( + remainingWaitMs(waitTimeoutMs, options.waitDeadline), + ), + }); const final = await waitUntilRunning(client, warehouseId, { - maxMs: PREFLIGHT_WAIT_MAX_MS, + maxMs: remainingWaitMs(waitTimeoutMs, options.waitDeadline), treatStoppedAsTransient: true, + lastKnownState: state, }); if (final === "RUNNING") { decision = "proceed"; @@ -701,7 +722,8 @@ export async function generateQueriesFromDescribe( } if (decision === "waitThenProceed") { const final = await waitUntilRunning(client, warehouseId, { - maxMs: PREFLIGHT_WAIT_MAX_MS, + maxMs: remainingWaitMs(waitTimeoutMs, options.waitDeadline), + lastKnownState: state, }); if (final === "RUNNING") { decision = "proceed"; @@ -713,7 +735,21 @@ export async function generateQueriesFromDescribe( } } } catch (err) { - if (isConnectivityError(err)) { + if (err instanceof WarehouseWaitTimeoutError) { + // The wait limit ran out with the warehouse still not RUNNING. This is + // environmental: the has-types gate falls back to committed types or + // fails with this detail. + decision = "degradeAll"; + isEnvironmental = true; + environmentalCause = "unavailable"; + environmentalDetail = describeWaitTimeout(err, waitTimeoutMs); + } else if (err instanceof WarehouseStatusTimeoutError) { + // The warehouse API itself didn't answer in time: same handling. + decision = "degradeAll"; + isEnvironmental = true; + environmentalCause = "unavailable"; + environmentalDetail = describeStatusTimeout(err); + } else if (isConnectivityError(err)) { // Warehouse unreachable (transient outage): degrade rather than fail — // never fail a build on a blip. Still environmental, so the caller's // has-types gate decides warn-and-fall-back (committed types present) @@ -1080,6 +1116,9 @@ export async function generateQueriesFromDescribe( environmentalCause: hadEnvironmentalFailure ? environmentalCause : undefined, + environmentalDetail: hadEnvironmentalFailure + ? environmentalDetail + : undefined, }; } diff --git a/packages/appkit/src/type-generator/tests/generate-queries.test.ts b/packages/appkit/src/type-generator/tests/generate-queries.test.ts index 2fe29a072..b836a0112 100644 --- a/packages/appkit/src/type-generator/tests/generate-queries.test.ts +++ b/packages/appkit/src/type-generator/tests/generate-queries.test.ts @@ -803,7 +803,10 @@ describe("generateQueriesFromDescribe", () => { // The stopped warehouse was started, then described once it came up. expect(mocks.startWarehouse).toHaveBeenCalledTimes(1); - expect(mocks.startWarehouse).toHaveBeenCalledWith({ id: "wh-123" }); + expect(mocks.startWarehouse).toHaveBeenCalledWith( + { id: "wh-123" }, + { signal: expect.any(AbortSignal) }, + ); expect(mocks.executeStatement).toHaveBeenCalledTimes(1); expect(schemas).toHaveLength(1); expect(schemas[0].name).toBe("a"); diff --git a/packages/appkit/src/type-generator/tests/index.test.ts b/packages/appkit/src/type-generator/tests/index.test.ts index 088b2bcf5..2cc00bcd3 100644 --- a/packages/appkit/src/type-generator/tests/index.test.ts +++ b/packages/appkit/src/type-generator/tests/index.test.ts @@ -773,12 +773,15 @@ describe("generateFromEntryPoint — metric-view emission", () => { expect(mocks.startWarehouse).toHaveBeenCalledWith( expect.anything(), "wh-1", + // Bounded like the status check, so a start that never answers can't hang. + { timeoutMs: 30_000 }, ); expect(mocks.waitUntilRunning).toHaveBeenCalledWith( expect.anything(), "wh-1", expect.objectContaining({ - maxMs: 300_000, + // The default 5-min budget, minus what the run already spent. + maxMs: expect.closeTo(300_000, -4), treatStoppedAsTransient: true, }), ); @@ -868,7 +871,7 @@ describe("generateFromEntryPoint — metric-view emission", () => { expect(mocks.waitUntilRunning).toHaveBeenCalledWith( expect.anything(), "wh-1", - expect.objectContaining({ maxMs: 300_000 }), + expect.objectContaining({ maxMs: expect.closeTo(300_000, -4) }), ); expect(mocks.executeStatement).not.toHaveBeenCalled(); // Degraded metric artifacts are NOT written in blocking mode. diff --git a/packages/appkit/src/type-generator/tests/warehouse-status.test.ts b/packages/appkit/src/type-generator/tests/warehouse-status.test.ts index 4a1b0f7b3..b479bc4ab 100644 --- a/packages/appkit/src/type-generator/tests/warehouse-status.test.ts +++ b/packages/appkit/src/type-generator/tests/warehouse-status.test.ts @@ -2,8 +2,16 @@ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import type { WorkspaceClient } from "../../workspace-client"; import { + DEFAULT_WAREHOUSE_WAIT_TIMEOUT_MS, + describeWaitTimeout, getWarehouseState, + resolveWarehouseWaitTimeoutMs, + startWarehouse, + statusRequestTimeoutMs, + WAREHOUSE_WAIT_TIMEOUT_ENV, type WarehouseState, + WarehouseStatusTimeoutError, + WarehouseWaitTimeoutError, waitUntilRunning, } from "../warehouse-status"; @@ -147,6 +155,140 @@ describe("waitUntilRunning", () => { await settled; }); + test("times out with a WarehouseWaitTimeoutError naming the warehouse, state and limit", async () => { + const get = vi.fn().mockResolvedValue(stateResponse("STARTING")); + const client = makeClient(get); + + const promise = waitUntilRunning(client, "wh-1", { maxMs: 3000 }); + const settled = promise.then( + () => undefined, + (error: unknown) => error, + ); + await vi.advanceTimersByTimeAsync(10000); + + const error = await settled; + expect(error).toBeInstanceOf(WarehouseWaitTimeoutError); + expect(error).toMatchObject({ + code: "warehouse_unavailable", + warehouseId: "wh-1", + lastState: "STARTING", + timeoutMs: 3000, + }); + expect(describeWaitTimeout(error as WarehouseWaitTimeoutError)).toBe( + "warehouse_unavailable: warehouse wh-1 is still STARTING after the 3s wait limit", + ); + }); + + test("never sleeps past the deadline: checks once more at the limit, then gives up", async () => { + const get = vi.fn().mockResolvedValue(stateResponse("STARTING")); + const client = makeClient(get); + + // Backoff would sleep 1000ms then 2000ms; with a 1500ms limit the second + // sleep is cut to 500ms so the wait ends at the limit, not at 3000ms. + const promise = waitUntilRunning(client, "wh-1", { maxMs: 1500 }); + let rejected = false; + promise.catch(() => { + rejected = true; + }); + + await vi.advanceTimersByTimeAsync(1500); + expect(rejected).toBe(true); + // t=0, t=1000 and a final check at t=1500. + expect(get).toHaveBeenCalledTimes(3); + }); + + test("a warehouse that comes up on the final check at the limit still resolves RUNNING", async () => { + const get = vi + .fn() + .mockResolvedValueOnce(stateResponse("STARTING")) + .mockResolvedValueOnce(stateResponse("RUNNING")); + const client = makeClient(get); + + const promise = waitUntilRunning(client, "wh-1", { maxMs: 500 }); + await vi.advanceTimersByTimeAsync(500); + + await expect(promise).resolves.toBe("RUNNING"); + }); + + test("a status request that never answers can't hold the wait past the limit", async () => { + // The reviewer's repro: STARTING once, then warehouses.get hangs forever. + let pendingSignal: AbortSignal | undefined; + const get = vi + .fn() + .mockResolvedValueOnce(stateResponse("STARTING")) + .mockImplementation( + (_req: unknown, options?: { signal?: AbortSignal }) => { + pendingSignal = options?.signal; + return new Promise(() => {}); + }, + ); + const client = makeClient(get); + + const promise = waitUntilRunning(client, "wh-1", { + maxMs: 20, + pollMs: 5, + }); + const settled = promise.then( + () => undefined, + (error: unknown) => error, + ); + await vi.advanceTimersByTimeAsync(20); + + const error = await settled; + expect(error).toBeInstanceOf(WarehouseWaitTimeoutError); + expect(error).toMatchObject({ lastState: "STARTING", timeoutMs: 20 }); + expect(get).toHaveBeenCalledTimes(2); + // The hung request was cancelled, not left running. + expect(pendingSignal?.aborted).toBe(true); + }); + + test("first poll never answers: reports the caller's last known state", async () => { + const get = vi.fn().mockImplementation(() => new Promise(() => {})); + const client = makeClient(get); + + const settled = waitUntilRunning(client, "wh-1", { + maxMs: 50, + lastKnownState: "STARTING", + }).then( + () => undefined, + (error: unknown) => error, + ); + await vi.advanceTimersByTimeAsync(50); + + const error = (await settled) as WarehouseWaitTimeoutError; + expect(error).toBeInstanceOf(WarehouseWaitTimeoutError); + expect(describeWaitTimeout(error)).toBe( + "warehouse_unavailable: warehouse wh-1 is still STARTING after the 50ms wait limit", + ); + }); + + test("no state ever observed: says the status request didn't answer", async () => { + const get = vi.fn().mockImplementation(() => new Promise(() => {})); + const client = makeClient(get); + + const settled = waitUntilRunning(client, "wh-1", { maxMs: 50 }).then( + () => undefined, + (error: unknown) => error, + ); + await vi.advanceTimersByTimeAsync(50); + + expect( + describeWaitTimeout((await settled) as WarehouseWaitTimeoutError), + ).toBe( + "warehouse_unavailable: warehouse wh-1 did not answer a status request after the 50ms wait limit", + ); + }); + + test("maxMs 0 checks once and does not wait", async () => { + const get = vi.fn().mockResolvedValue(stateResponse("STARTING")); + const client = makeClient(get); + + await expect( + waitUntilRunning(client, "wh-1", { maxMs: 0 }), + ).rejects.toBeInstanceOf(WarehouseWaitTimeoutError); + expect(get).toHaveBeenCalledTimes(1); + }); + test("stops immediately when the signal is already aborted", async () => { const get = vi.fn().mockResolvedValue(stateResponse("STARTING")); const client = makeClient(get); @@ -193,3 +335,96 @@ describe("waitUntilRunning", () => { expect(get).toHaveBeenCalledTimes(1); }); }); + +describe("bounded warehouse API calls", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + test("getWarehouseState with timeoutMs rejects and aborts a request that never answers", async () => { + let signal: AbortSignal | undefined; + const get = vi.fn((_req: unknown, options?: { signal?: AbortSignal }) => { + signal = options?.signal; + return new Promise(() => {}); + }); + const settled = getWarehouseState(makeClient(get), "wh-1", { + timeoutMs: 1000, + }).then( + () => undefined, + (error: unknown) => error, + ); + await vi.advanceTimersByTimeAsync(1000); + + const error = await settled; + expect(error).toBeInstanceOf(WarehouseStatusTimeoutError); + expect(error).toMatchObject({ warehouseId: "wh-1", request: "status" }); + expect(signal?.aborted).toBe(true); + }); + + test("getWarehouseState with timeoutMs returns an answer that arrives in time", async () => { + const get = vi.fn().mockResolvedValue(stateResponse("RUNNING")); + await expect( + getWarehouseState(makeClient(get), "wh-1", { timeoutMs: 1000 }), + ).resolves.toBe("RUNNING"); + }); + + test("startWarehouse with timeoutMs rejects a start request that never answers", async () => { + const start = vi.fn(() => new Promise(() => {})); + const client = { + warehouses: { startWarehouse: start }, + } as unknown as WorkspaceClient; + const settled = startWarehouse(client, "wh-1", { timeoutMs: 1000 }).then( + () => undefined, + (error: unknown) => error, + ); + await vi.advanceTimersByTimeAsync(1000); + + expect(await settled).toMatchObject({ + name: "WarehouseStatusTimeoutError", + request: "start", + }); + }); + + test("statusRequestTimeoutMs keeps the preflight check between 10s and 30s", () => { + expect(statusRequestTimeoutMs(0)).toBe(10_000); + expect(statusRequestTimeoutMs(20_000)).toBe(20_000); + expect(statusRequestTimeoutMs(300_000)).toBe(30_000); + }); +}); + +describe("resolveWarehouseWaitTimeoutMs", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + test("defaults to 5 minutes", () => { + vi.stubEnv(WAREHOUSE_WAIT_TIMEOUT_ENV, ""); + expect(resolveWarehouseWaitTimeoutMs()).toBe(300_000); + expect(DEFAULT_WAREHOUSE_WAIT_TIMEOUT_MS).toBe(300_000); + }); + + test("reads the env var in seconds", () => { + vi.stubEnv(WAREHOUSE_WAIT_TIMEOUT_ENV, "90"); + expect(resolveWarehouseWaitTimeoutMs()).toBe(90_000); + }); + + test("an explicit value wins over the env var", () => { + vi.stubEnv(WAREHOUSE_WAIT_TIMEOUT_ENV, "90"); + expect(resolveWarehouseWaitTimeoutMs(5_000)).toBe(5_000); + expect(resolveWarehouseWaitTimeoutMs(0)).toBe(0); + }); + + test("ignores an invalid env value, reports it, and uses the default", () => { + const onInvalid = vi.fn(); + vi.stubEnv(WAREHOUSE_WAIT_TIMEOUT_ENV, "five minutes"); + expect(resolveWarehouseWaitTimeoutMs(undefined, onInvalid)).toBe(300_000); + expect(onInvalid).toHaveBeenCalledWith("five minutes"); + + vi.stubEnv(WAREHOUSE_WAIT_TIMEOUT_ENV, "-1"); + expect(resolveWarehouseWaitTimeoutMs()).toBe(300_000); + }); +}); diff --git a/packages/appkit/src/type-generator/tests/warehouse-wait-limit.test.ts b/packages/appkit/src/type-generator/tests/warehouse-wait-limit.test.ts new file mode 100644 index 000000000..c07cc76a8 --- /dev/null +++ b/packages/appkit/src/type-generator/tests/warehouse-wait-limit.test.ts @@ -0,0 +1,318 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { + afterAll, + afterEach, + beforeEach, + describe, + expect, + test, + vi, +} from "vitest"; + +/** + * The `--wait` time limit, end to end through the real query and metric-view + * paths and the real `waitUntilRunning` loop. Only the SDK client is stubbed. + */ + +const mocks = vi.hoisted(() => ({ + getWarehouse: vi.fn(), + startWarehouse: vi.fn(), + executeStatement: vi.fn(), +})); + +vi.mock("../../workspace-client", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + createWorkspaceClient: () => ({ + statementExecution: { executeStatement: mocks.executeStatement }, + warehouses: { + getWarehouse: mocks.getWarehouse, + startWarehouse: mocks.startWarehouse, + }, + }), + }; +}); + +// Start every test from an empty typegen cache so each query is described (or +// degraded) rather than served from a previous run. +vi.mock("../cache", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + loadCache: vi.fn(async () => ({ + version: actual.CACHE_VERSION, + queries: {}, + })), + saveCache: vi.fn(), + }; +}); + +const { generateFromEntryPoint, TypegenFatalError } = await import("../index"); + +const testDir = path.join(__dirname, "__output_wait_limit__"); +const queryFolder = path.join(testDir, "queries"); +const metricViewsFolder = path.join(testDir, "metric-views"); +const outFile = path.join(testDir, "generated", "analytics.d.ts"); +const metricFile = path.join(testDir, "generated", "metric-views.d.ts"); + +const stuck = () => mocks.getWarehouse.mockResolvedValue({ state: "STARTING" }); + +function describeQueryResult() { + return { + statementId: "stmt-1", + status: { state: "SUCCEEDED" }, + result: { dataArray: [["id", "BIGINT", null]] }, + }; +} + +function writeCommitted(file: string, contents: string) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, contents, "utf-8"); +} + +function writeMetricConfig() { + fs.mkdirSync(metricViewsFolder, { recursive: true }); + fs.writeFileSync( + path.join(metricViewsFolder, "definitions.json"), + JSON.stringify({ + metricViews: { revenue: { source: "demo.sales.revenue" } }, + }), + "utf-8", + ); +} + +async function rejection(promise: Promise): Promise { + return promise.then( + () => undefined, + (error: unknown) => error, + ); +} + +function gateWarnings(spy: ReturnType): string { + return spy.mock.calls + .flat() + .map(String) + .filter((line) => line.includes("AppKit typegen")) + .join("\n"); +} + +describe("generate-types --wait: warehouse wait limit", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.unstubAllEnvs(); + fs.rmSync(testDir, { recursive: true, force: true }); + fs.mkdirSync(queryFolder, { recursive: true }); + fs.writeFileSync( + path.join(queryFolder, "users.sql"), + "SELECT id FROM users", + "utf-8", + ); + mocks.executeStatement.mockResolvedValue(describeQueryResult()); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + afterAll(() => { + fs.rmSync(testDir, { recursive: true, force: true }); + }); + + test("running warehouse: describes immediately, no waiting", async () => { + mocks.getWarehouse.mockResolvedValue({ state: "RUNNING" }); + + await generateFromEntryPoint({ + outFile, + queryFolder, + warehouseId: "wh-1", + mode: "blocking", + waitTimeoutMs: 50, + }); + + expect(mocks.getWarehouse).toHaveBeenCalledTimes(1); + expect(mocks.executeStatement).toHaveBeenCalledTimes(1); + expect(fs.readFileSync(outFile, "utf-8")).toContain("id: number"); + }); + + test("warehouse becomes ready within the limit: waits, then writes real types", async () => { + mocks.getWarehouse + .mockResolvedValueOnce({ state: "STARTING" }) + .mockResolvedValue({ state: "RUNNING" }); + + await generateFromEntryPoint({ + outFile, + queryFolder, + warehouseId: "wh-1", + mode: "blocking", + waitTimeoutMs: 5_000, + }); + + // Preflight probe (STARTING), then the wait loop's poll that sees RUNNING. + expect(mocks.getWarehouse).toHaveBeenCalledTimes(2); + expect(mocks.executeStatement).toHaveBeenCalledTimes(1); + const types = fs.readFileSync(outFile, "utf-8"); + expect(types).toContain("id: number"); + expect(types).not.toContain("result: unknown"); + }); + + test("limit reached with committed types: keeps them and warns with warehouse, state and limit", async () => { + stuck(); + const committed = "// committed types\n"; + writeCommitted(outFile, committed); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + try { + await expect( + generateFromEntryPoint({ + outFile, + queryFolder, + warehouseId: "wh-stuck", + mode: "blocking", + waitTimeoutMs: 50, + }), + ).resolves.toBeUndefined(); + + const warnings = gateWarnings(warnSpy); + expect(warnings).toContain("AppKit typegen: using committed types"); + expect(warnings).toContain( + "warehouse_unavailable: warehouse wh-stuck is still STARTING after the 50ms wait limit", + ); + expect(warnings).toContain("kept existing analytics.d.ts"); + } finally { + warnSpy.mockRestore(); + } + + expect(fs.readFileSync(outFile, "utf-8")).toBe(committed); + expect(mocks.executeStatement).not.toHaveBeenCalled(); + }); + + test("limit reached without committed types: fails with warehouse, state, limit and the knob", async () => { + stuck(); + + const error = await rejection( + generateFromEntryPoint({ + outFile, + queryFolder, + warehouseId: "wh-stuck", + mode: "blocking", + waitTimeoutMs: 50, + }), + ); + + expect(error).toBeInstanceOf(TypegenFatalError); + const message = (error as Error).message; + expect(message).toContain( + "warehouse_unavailable: warehouse wh-stuck is still STARTING after the 50ms wait limit", + ); + expect(message).toContain("no committed types to fall back on"); + expect(message).toContain("No type files were written."); + expect(message).not.toContain("result: unknown"); + expect((error as { code?: string }).code).toBe("warehouse_unavailable"); + expect(message).toContain("analytics.d.ts"); + expect(message).toContain("--wait-timeout"); + expect(message).toContain("APPKIT_TYPEGEN_WAIT_TIMEOUT"); + expect(fs.existsSync(outFile)).toBe(false); + expect(mocks.executeStatement).not.toHaveBeenCalled(); + }); + + test("a stopped warehouse that never comes up is started, then times out the same way", async () => { + mocks.getWarehouse + .mockResolvedValueOnce({ state: "STOPPED" }) + .mockResolvedValue({ state: "STARTING" }); + + const error = await rejection( + generateFromEntryPoint({ + outFile, + queryFolder, + warehouseId: "wh-cold", + mode: "blocking", + waitTimeoutMs: 50, + }), + ); + + expect(mocks.startWarehouse).toHaveBeenCalledWith( + { id: "wh-cold" }, + { signal: expect.any(AbortSignal) }, + ); + expect(error).toBeInstanceOf(TypegenFatalError); + expect((error as Error).message).toContain( + "warehouse wh-cold is still STARTING after the 50ms wait limit", + ); + }); + + test("a status request that hangs mid-wait still ends at the limit", async () => { + // Preflight says STARTING, then every warehouses.get hangs. + mocks.getWarehouse + .mockResolvedValueOnce({ state: "STARTING" }) + .mockImplementation(() => new Promise(() => {})); + + const started = Date.now(); + const error = await rejection( + generateFromEntryPoint({ + outFile, + queryFolder, + warehouseId: "wh-hung", + mode: "blocking", + waitTimeoutMs: 50, + }), + ); + + expect(Date.now() - started).toBeLessThan(1_000); + expect(error).toBeInstanceOf(TypegenFatalError); + expect((error as Error).message).toContain( + "warehouse_unavailable: warehouse wh-hung is still STARTING after the 50ms wait limit", + ); + }); + + test("APPKIT_TYPEGEN_WAIT_TIMEOUT sets the limit in seconds", async () => { + stuck(); + vi.stubEnv("APPKIT_TYPEGEN_WAIT_TIMEOUT", "0.05"); + + const started = Date.now(); + const error = await rejection( + generateFromEntryPoint({ + outFile, + queryFolder, + warehouseId: "wh-stuck", + mode: "blocking", + }), + ); + + expect(Date.now() - started).toBeLessThan(2_000); + expect((error as Error).message).toContain("after the 50ms wait limit"); + }); + + test("queries and metric views share one wait budget", async () => { + stuck(); + writeMetricConfig(); + + const started = Date.now(); + const error = await rejection( + generateFromEntryPoint({ + outFile, + queryFolder, + metricViewsFolder, + mvOutFile: metricFile, + warehouseId: "wh-stuck", + mode: "blocking", + waitTimeoutMs: 300, + }), + ); + const elapsed = Date.now() - started; + + // One 300ms budget for the run, not 300ms per path. + expect(elapsed).toBeLessThan(550); + expect(error).toBeInstanceOf(TypegenFatalError); + const message = (error as Error).message; + expect(message).toContain( + "warehouse wh-stuck is still STARTING after the 300ms wait limit", + ); + expect(message).toContain("analytics.d.ts"); + expect(message).toContain("metric-views.d.ts"); + expect(mocks.executeStatement).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/appkit/src/type-generator/types.ts b/packages/appkit/src/type-generator/types.ts index 124a990c5..cf03d3707 100644 --- a/packages/appkit/src/type-generator/types.ts +++ b/packages/appkit/src/type-generator/types.ts @@ -164,4 +164,6 @@ export interface QueryGenerationResult { fatalErrors: QueryFatalError[]; hadEnvironmentalFailure?: boolean; environmentalCause?: "auth" | "unreachable" | "unavailable"; + /** Specific reason for the environmental failure (e.g. wait limit reached). */ + environmentalDetail?: string; } diff --git a/packages/appkit/src/type-generator/warehouse-status.ts b/packages/appkit/src/type-generator/warehouse-status.ts index 8aae70e5e..a161edfc6 100644 --- a/packages/appkit/src/type-generator/warehouse-status.ts +++ b/packages/appkit/src/type-generator/warehouse-status.ts @@ -17,6 +17,225 @@ export type WarehouseState = const INITIAL_POLL_MS = 1000; const MAX_POLL_MS = 15000; +/** + * Default limit (5 min) on how long a `--wait` run waits for the SQL warehouse + * to reach RUNNING. Long enough for a typical cold start; short enough that a + * warehouse stuck in STARTING can't hang a build. + */ +export const DEFAULT_WAREHOUSE_WAIT_TIMEOUT_MS = 300_000; + +/** + * Env var that sets the warehouse wait limit, in seconds. The CLI's + * `--wait-timeout ` flag takes precedence over it. + */ +export const WAREHOUSE_WAIT_TIMEOUT_ENV = "APPKIT_TYPEGEN_WAIT_TIMEOUT"; + +/** + * Error code shared by every surface that reports a warehouse that could not + * serve type generation in time (typegen, deploy checks, deploy docs). + */ +export const WAREHOUSE_UNAVAILABLE_CODE = "warehouse_unavailable"; + +/** + * Cap on a single `warehouses.get` made outside a wait (the preflight status + * check), so an API that never answers can't hang the run either. Polls inside + * {@link waitUntilRunning} are bounded by the remaining wait budget instead. + */ +const STATUS_REQUEST_TIMEOUT_MS = 30_000; + +/** + * Lower bound for the preflight status check, so a very short (or `0`) wait + * limit still leaves time for one answer from the API. + */ +const MIN_STATUS_REQUEST_TIMEOUT_MS = 10_000; + +/** + * Time allowed for the preflight status check given the wait budget left: + * the budget, but at least {@link MIN_STATUS_REQUEST_TIMEOUT_MS} and at most + * {@link STATUS_REQUEST_TIMEOUT_MS}. + */ +export function statusRequestTimeoutMs(remainingMs: number): number { + return Math.min( + STATUS_REQUEST_TIMEOUT_MS, + Math.max(remainingMs, MIN_STATUS_REQUEST_TIMEOUT_MS), + ); +} + +/** + * Thrown by {@link getWarehouseState} / {@link startWarehouse} when the + * warehouse API doesn't answer within `timeoutMs`. The request is aborted. + */ +export class WarehouseStatusTimeoutError extends Error { + readonly code = WAREHOUSE_UNAVAILABLE_CODE; + readonly warehouseId: string; + readonly timeoutMs: number; + readonly request: "status" | "start"; + + constructor( + warehouseId: string, + timeoutMs: number, + request: "status" | "start" = "status", + ) { + super( + `Warehouse ${warehouseId} did not answer a ${request} request within ${timeoutMs}ms`, + ); + this.name = "WarehouseStatusTimeoutError"; + this.warehouseId = warehouseId; + this.timeoutMs = timeoutMs; + this.request = request; + } +} + +/** One-line, user-facing description of a warehouse API request timeout. */ +export function describeStatusTimeout( + error: WarehouseStatusTimeoutError, +): string { + return `${WAREHOUSE_UNAVAILABLE_CODE}: warehouse ${error.warehouseId} did not answer a ${error.request} request within ${formatWaitLimit(error.timeoutMs)}`; +} + +/** + * Run a warehouse API call that must settle within `timeoutMs`. The call gets + * an AbortSignal (aborted on timeout or when `signal` aborts); the deadline is + * enforced here as well, so it holds even if a transport ignores the signal. + */ +async function withRequestDeadline( + call: (signal: AbortSignal) => Promise, + opts: { + warehouseId: string; + request: "status" | "start"; + timeoutMs?: number; + signal?: AbortSignal; + }, +): Promise { + const { timeoutMs, signal } = opts; + const controller = new AbortController(); + const onAbort = () => controller.abort(signal?.reason); + if (signal?.aborted) onAbort(); + else signal?.addEventListener("abort", onAbort, { once: true }); + + let timer: ReturnType | undefined; + const timedOut = new Promise((_, reject) => { + if (timeoutMs === undefined) return; + timer = setTimeout( + () => { + controller.abort(); + reject( + new WarehouseStatusTimeoutError( + opts.warehouseId, + timeoutMs, + opts.request, + ), + ); + }, + Math.max(0, timeoutMs), + ); + }); + + // Async wrapper: a synchronous throw or a non-promise result still settles + // the race like any other answer. + const request = (async () => call(controller.signal))(); + // Once the deadline wins, the aborted request's rejection is expected noise. + request.catch(() => {}); + + try { + return await Promise.race([request, timedOut]); + } finally { + clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + } +} + +/** + * Thrown by {@link waitUntilRunning} when the warehouse does not reach RUNNING + * within the wait limit. Carries what the user needs to act on: the warehouse, + * the last state it reported (if any) and the limit that ran out. + */ +export class WarehouseWaitTimeoutError extends Error { + readonly code = WAREHOUSE_UNAVAILABLE_CODE; + readonly warehouseId: string; + /** Last state reported; `undefined` if no status request was answered. */ + readonly lastState: WarehouseState | undefined; + readonly timeoutMs: number; + + constructor( + warehouseId: string, + lastState: WarehouseState | undefined, + timeoutMs: number, + ) { + super( + `Warehouse ${warehouseId} did not reach RUNNING within ${timeoutMs}ms (last state: ${lastState ?? "unknown, status request did not answer"})`, + ); + this.name = "WarehouseWaitTimeoutError"; + this.warehouseId = warehouseId; + this.lastState = lastState; + this.timeoutMs = timeoutMs; + } +} + +/** + * One-line, user-facing description of a wait timeout, prefixed with the + * shared {@link WAREHOUSE_UNAVAILABLE_CODE} so logs can be matched on it. + * `limitMs` is the limit to report — the full configured limit, which can be + * larger than the remainder a later wait in the same run was given. + */ +export function describeWaitTimeout( + error: WarehouseWaitTimeoutError, + limitMs: number = error.timeoutMs, +): string { + const state = error.lastState + ? `is still ${error.lastState}` + : "did not answer a status request"; + return `${WAREHOUSE_UNAVAILABLE_CODE}: warehouse ${error.warehouseId} ${state} after the ${formatWaitLimit(limitMs)} wait limit`; +} + +/** + * Milliseconds left before `deadline` (epoch ms), or `limitMs` when no shared + * deadline was given. Lets one `--wait` run spend a single wait budget across + * the query and metric-view paths instead of one budget each. + */ +export function remainingWaitMs(limitMs: number, deadline?: number): number { + if (deadline === undefined) return limitMs; + return Math.max(0, deadline - Date.now()); +} + +/** Format a millisecond limit for humans: whole seconds when it divides evenly. */ +function formatWaitLimit(ms: number): string { + return ms % 1000 === 0 ? `${ms / 1000}s` : `${ms}ms`; +} + +/** + * Resolve the warehouse wait limit in milliseconds. + * + * Precedence: an explicit `waitTimeoutMs` (the CLI flag, or a library caller), + * then {@link WAREHOUSE_WAIT_TIMEOUT_ENV} (seconds), then + * {@link DEFAULT_WAREHOUSE_WAIT_TIMEOUT_MS}. `0` is valid and means "check + * once, don't wait". An invalid env value is ignored with a warning rather + * than failing the build. + */ +export function resolveWarehouseWaitTimeoutMs( + waitTimeoutMs?: number, + onInvalidEnv?: (raw: string) => void, +): number { + if ( + typeof waitTimeoutMs === "number" && + Number.isFinite(waitTimeoutMs) && + waitTimeoutMs >= 0 + ) { + return Math.floor(waitTimeoutMs); + } + + const raw = process.env[WAREHOUSE_WAIT_TIMEOUT_ENV]?.trim(); + if (raw) { + const seconds = Number(raw); + if (Number.isFinite(seconds) && seconds >= 0) { + return Math.floor(seconds * 1000); + } + onInvalidEnv?.(raw); + } + + return DEFAULT_WAREHOUSE_WAIT_TIMEOUT_MS; +} + /** States from which the warehouse will not transition to RUNNING on its own. */ const NOT_COMING_UP: ReadonlySet = new Set([ "STOPPED", @@ -64,14 +283,27 @@ function delay(ms: number, signal?: AbortSignal): Promise { /** * Fetch the current lifecycle state of a SQL warehouse. * + * Pass `opts.timeoutMs` to bound the request: if it hasn't answered by then it + * is aborted and a {@link WarehouseStatusTimeoutError} is thrown. The bound is + * enforced here, not just handed to the SDK, so it holds even if a transport + * ignores the signal. `opts.signal` cancels the request too. + * * Errors from the SDK (auth, bad warehouse id, connectivity) are intentionally * NOT caught — the caller decides how to classify and react to them. */ export async function getWarehouseState( client: WorkspaceClient, warehouseId: string, + opts: { timeoutMs?: number; signal?: AbortSignal } = {}, ): Promise { - const response = await client.warehouses.getWarehouse({ id: warehouseId }); + if (opts.timeoutMs === undefined && opts.signal === undefined) { + const response = await client.warehouses.getWarehouse({ id: warehouseId }); + return response.state as WarehouseState; + } + const response = await withRequestDeadline( + (signal) => client.warehouses.getWarehouse({ id: warehouseId }, { signal }), + { warehouseId, request: "status", ...opts }, + ); return response.state as WarehouseState; } @@ -83,14 +315,25 @@ export async function getWarehouseState( * job here — {@link waitUntilRunning} is the poller that watches the warehouse * the rest of the way to RUNNING. We just nudge it out of the stopped state. * + * Pass `opts.timeoutMs` to bound the request (see {@link getWarehouseState}). + * * Errors from the SDK (auth, bad warehouse id, connectivity) are intentionally * NOT caught — the caller decides how to classify and react to them. */ export async function startWarehouse( client: WorkspaceClient, warehouseId: string, + opts: { timeoutMs?: number } = {}, ): Promise { - await client.warehouses.startWarehouse({ id: warehouseId }); + if (opts.timeoutMs === undefined) { + await client.warehouses.startWarehouse({ id: warehouseId }); + return; + } + await withRequestDeadline( + (signal) => + client.warehouses.startWarehouse({ id: warehouseId }, { signal }), + { warehouseId, request: "start", timeoutMs: opts.timeoutMs }, + ); } /** @@ -116,7 +359,12 @@ export async function startWarehouse( * down): the next deadline/abort check throws an `AbortError`, and a pending * backoff sleep resolves immediately rather than holding the process open. * - * @throws Error if `maxMs` elapses before the warehouse reaches RUNNING. + * Backoff sleeps never run past the deadline, and each `warehouses.get` is + * bounded by the budget left, so the wait settles within `maxMs` even when a + * status request never answers. + * + * @throws WarehouseWaitTimeoutError if `maxMs` elapses before the warehouse + * reaches RUNNING. * @throws Error (`name === "AbortError"`) if `opts.signal` is or becomes aborted. */ export async function waitUntilRunning( @@ -127,11 +375,14 @@ export async function waitUntilRunning( pollMs?: number; signal?: AbortSignal; treatStoppedAsTransient?: boolean; + /** State the caller already observed, reported if no poll answers. */ + lastKnownState?: WarehouseState; }, ): Promise { const { maxMs, signal, treatStoppedAsTransient } = opts; const start = Date.now(); let pollMs = opts.pollMs ?? INITIAL_POLL_MS; + let lastState = opts.lastKnownState; // Which states end the wait early. When we've just issued a start, STOPPED and // STOPPING are expected stale readings, so only DELETED/DELETING stay terminal. @@ -142,27 +393,34 @@ export async function waitUntilRunning( while (true) { throwIfAborted(signal); - const state = await getWarehouseState(client, warehouseId); + // Each poll gets only what's left of the budget, so a status request that + // never answers can't hold the wait past the limit. + let state: WarehouseState; + try { + state = await getWarehouseState(client, warehouseId, { + timeoutMs: Math.max(0, maxMs - (Date.now() - start)), + signal, + }); + } catch (error) { + throwIfAborted(signal); + if (error instanceof WarehouseStatusTimeoutError) { + throw new WarehouseWaitTimeoutError(warehouseId, lastState, maxMs); + } + throw error; + } + lastState = state; if (state === "RUNNING") return "RUNNING"; if (terminalStates.has(state)) return state; - if (Date.now() - start >= maxMs) { - throw new Error( - `Warehouse ${warehouseId} did not reach RUNNING within ${maxMs}ms (last state: ${state})`, - ); + const remaining = maxMs - (Date.now() - start); + if (remaining <= 0) { + throw new WarehouseWaitTimeoutError(warehouseId, state, maxMs); } - await delay(pollMs, signal); + // Never sleep past the deadline: wake at it and make one last check. + await delay(Math.min(pollMs, remaining), signal); throwIfAborted(signal); - // Re-check the deadline after sleeping so we don't issue another poll past - // the budget purely because we napped through it. - if (Date.now() - start >= maxMs) { - throw new Error( - `Warehouse ${warehouseId} did not reach RUNNING within ${maxMs}ms (last state: ${state})`, - ); - } - pollMs = Math.min(pollMs * 2, MAX_POLL_MS); } } diff --git a/packages/shared/src/cli/commands/generate-types.test.ts b/packages/shared/src/cli/commands/generate-types.test.ts index 9223d960c..b5bb02280 100644 --- a/packages/shared/src/cli/commands/generate-types.test.ts +++ b/packages/shared/src/cli/commands/generate-types.test.ts @@ -69,9 +69,15 @@ vi.mock("./spawn-lock.js", () => ({ releaseSpawnLock, getSpawnLockPath, SPAWN_LOCK_STALE_MS: 360_000, + spawnLockStaleMsFor: (ms?: number) => + ms === undefined ? 360_000 : Math.max(360_000, ms + 60_000), })); -import { generateTypesCommand, resolveTypegenMode } from "./generate-types"; +import { + generateTypesCommand, + parseWaitTimeoutSeconds, + resolveTypegenMode, +} from "./generate-types"; /** * Drive the real commander command the way the bin does, so argv parsing @@ -103,6 +109,20 @@ describe("resolveTypegenMode (generate-types --wait)", () => { }); }); +describe("parseWaitTimeoutSeconds (--wait-timeout)", () => { + test("accepts whole and fractional seconds, and 0", () => { + expect(parseWaitTimeoutSeconds("300")).toBe(300); + expect(parseWaitTimeoutSeconds("0.5")).toBe(0.5); + expect(parseWaitTimeoutSeconds("0")).toBe(0); + }); + + test.each(["", "abc", "-1", "Infinity"])("rejects %j", (value) => { + expect(() => parseWaitTimeoutSeconds(value)).toThrow( + "Expected a number of seconds", + ); + }); +}); + describe("generate-types foreground spawn orchestration", () => { let tmpRoot: string; let consoleLog: Mock; @@ -112,6 +132,10 @@ describe("generate-types foreground spawn orchestration", () => { beforeEach(() => { vi.clearAllMocks(); acquireSpawnLock.mockReturnValue(true); + // The command is a module singleton and commander keeps parsed option + // values between parseAsync calls; clear the ones these tests vary. + generateTypesCommand.setOptionValue("wait", undefined); + generateTypesCommand.setOptionValue("waitTimeout", undefined); // A real temp project root with config/queries and config/metric-views // folders so the analytics + metric generate paths run. @@ -240,10 +264,46 @@ describe("generate-types foreground spawn orchestration", () => { expect(spawn).not.toHaveBeenCalled(); }); + test("--wait-timeout is passed to the generator in milliseconds", async () => { + await runCli([tmpRoot, "--wait", "--wait-timeout", "120"]); + + expect(generateFromEntryPoint).toHaveBeenCalledWith( + expect.objectContaining({ mode: "blocking", waitTimeoutMs: 120_000 }), + ); + }); + + test("without --wait-timeout the generator resolves the limit itself", async () => { + await runCli([tmpRoot, "--wait"]); + + expect(generateFromEntryPoint).toHaveBeenCalledWith( + expect.objectContaining({ waitTimeoutMs: undefined }), + ); + }); + + test("a long --wait-timeout stretches the worker lock's staleness window", async () => { + await runCli([tmpRoot, "--wait-timeout", "900"]); + + expect(acquireSpawnLock).toHaveBeenCalledWith( + getSpawnLockPath(tmpRoot), + 960_000, + ); + }); + + test("the background worker inherits --wait-timeout", async () => { + await runCli([tmpRoot, "--wait-timeout", "45"]); + + expect(spawn).toHaveBeenCalledTimes(1); + const argv = spawn.mock.calls[0][1] as string[]; + const at = argv.indexOf("--wait-timeout"); + expect(at).toBeGreaterThan(-1); + expect(argv[at + 1]).toBe("45"); + }); + test("worker invocation (--worker-lock): runs blocking generate, releases lock, does NOT spawn", async () => { const lockPath = getSpawnLockPath(tmpRoot); - await runCli([tmpRoot, "--worker-lock", lockPath]); + // Same flags spawnTypegenWorker passes: a worker is always `--wait`. + await runCli([tmpRoot, "--wait", "--worker-lock", lockPath]); // A worker is always blocking — it does the real DESCRIBE lifecycle. expect(generateFromEntryPoint).toHaveBeenCalledWith( diff --git a/packages/shared/src/cli/commands/generate-types.ts b/packages/shared/src/cli/commands/generate-types.ts index cbabbd097..e547495fc 100644 --- a/packages/shared/src/cli/commands/generate-types.ts +++ b/packages/shared/src/cli/commands/generate-types.ts @@ -2,15 +2,22 @@ import { spawn } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; -import { Command, Option } from "commander"; +import { Command, InvalidArgumentError, Option } from "commander"; import { METRIC_CONFIG_FILE } from "../../schemas/metric-fqn"; import { acquireSpawnLock, getSpawnLockPath, releaseSpawnLock, + spawnLockStaleMsFor, } from "./spawn-lock.js"; +/** + * Env var for the warehouse wait limit, in seconds. Read by the type generator; + * the CLI only reads it to size the background worker's lock. + */ +const WAIT_TIMEOUT_ENV = "APPKIT_TYPEGEN_WAIT_TIMEOUT"; + /** * Resolve the typegen pre-flight mode for the CLI. Defaults to "non-blocking" — * a one-shot CLI can't describe in the background, so by default it never @@ -27,10 +34,40 @@ export function resolveTypegenMode(options?: { return options?.wait ? "blocking" : "non-blocking"; } +/** + * Parse `--wait-timeout `: a non-negative number of seconds. `0` means + * "check the warehouse once, don't wait". + */ +export function parseWaitTimeoutSeconds(value: string): number { + const seconds = Number(value); + if (value.trim() === "" || !Number.isFinite(seconds) || seconds < 0) { + throw new InvalidArgumentError("Expected a number of seconds (0 or more)."); + } + return seconds; +} + +/** + * The wait limit the background worker will use, in milliseconds, when one is + * configured (flag first, then env). `undefined` means the generator default. + */ +function configuredWaitTimeoutMs(flagSeconds?: number): number | undefined { + if (flagSeconds !== undefined) return flagSeconds * 1000; + const raw = process.env[WAIT_TIMEOUT_ENV]?.trim(); + if (!raw) return undefined; + const seconds = Number(raw); + return Number.isFinite(seconds) && seconds >= 0 ? seconds * 1000 : undefined; +} + /** Options parsed by commander for the generate-types command. */ interface GenerateTypesOptions { noCache?: boolean; wait?: boolean; + /** + * `--wait-timeout `: limit on waiting for the warehouse to reach + * RUNNING under `--wait`. Unset → `APPKIT_TYPEGEN_WAIT_TIMEOUT`, else 5 min + * (resolved by the type generator). + */ + waitTimeout?: number; /** * Internal: present only on the detached worker invocation. Carries the path * of the single-flight lock this worker must release when it finishes. Its @@ -85,6 +122,10 @@ async function runGenerateTypes( warehouseId: resolvedWarehouseId, noCache, mode, + waitTimeoutMs: + options?.waitTimeout === undefined + ? undefined + : options.waitTimeout * 1000, }); if (hasQueries) { @@ -183,7 +224,12 @@ async function runGenerateTypes( */ export function spawnTypegenWorker( lockPath: string, - targets: { rootDir?: string; outFile?: string; warehouseId?: string }, + targets: { + rootDir?: string; + outFile?: string; + warehouseId?: string; + waitTimeout?: number; + }, ): boolean { // The script the runtime launched us with (the `appkit` bin shim). Re-running // it under the same node binary reproduces this exact CLI in the worker. @@ -212,6 +258,9 @@ export function spawnTypegenWorker( "--wait", "--worker-lock", lockPath, + ...(targets.waitTimeout === undefined + ? [] + : ["--wait-timeout", String(targets.waitTimeout)]), ...positionals, ]; @@ -276,8 +325,16 @@ async function generateTypesAction( const resolvedRootDir = rootDir || process.cwd(); const lockPath = getSpawnLockPath(resolvedRootDir); - if (acquireSpawnLock(lockPath)) { - spawnTypegenWorker(lockPath, { rootDir, outFile, warehouseId }); + const staleMs = spawnLockStaleMsFor( + configuredWaitTimeoutMs(options.waitTimeout), + ); + if (acquireSpawnLock(lockPath, staleMs)) { + spawnTypegenWorker(lockPath, { + rootDir, + outFile, + warehouseId, + waitTimeout: options.waitTimeout, + }); } else { console.log("Type refresh already in progress, skipping."); } @@ -298,6 +355,11 @@ export const generateTypesCommand = new Command("generate-types") "--wait", "Wait for warehouse readiness instead of degrading (use for CI)", ) + .option( + "--wait-timeout ", + "With --wait: give up waiting for the warehouse after this many seconds, then keep committed types or fail (default: $APPKIT_TYPEGEN_WAIT_TIMEOUT, else 300)", + parseWaitTimeoutSeconds, + ) // Internal: marks the detached background worker and carries the lock it must // release. Hidden from --help; users should never pass it. .addOption( @@ -314,6 +376,7 @@ Examples: $ appkit generate-types . shared/appkit-types/analytics.d.ts $ appkit generate-types . shared/appkit-types/analytics.d.ts my-warehouse-id $ appkit generate-types --no-cache - $ appkit generate-types --wait # CI: wait for the warehouse and fail on a cold one`, + $ appkit generate-types --wait # CI: wait for the warehouse and fail on a cold one + $ appkit generate-types --wait --wait-timeout 120 # give up on the warehouse after 2 minutes`, ) .action(generateTypesAction); diff --git a/packages/shared/src/cli/commands/spawn-lock.test.ts b/packages/shared/src/cli/commands/spawn-lock.test.ts index fef2a45a4..059b1fbc4 100644 --- a/packages/shared/src/cli/commands/spawn-lock.test.ts +++ b/packages/shared/src/cli/commands/spawn-lock.test.ts @@ -9,6 +9,7 @@ import { getSpawnLockPath, releaseSpawnLock, SPAWN_LOCK_STALE_MS, + spawnLockStaleMsFor, } from "./spawn-lock"; describe("spawn-lock", () => { @@ -93,3 +94,14 @@ describe("spawn-lock", () => { expect(acquireSpawnLock(lockPath)).toBe(true); // freed → re-acquirable }); }); + +describe("spawnLockStaleMsFor", () => { + test("keeps the default window for the default or a shorter wait limit", () => { + expect(spawnLockStaleMsFor()).toBe(SPAWN_LOCK_STALE_MS); + expect(spawnLockStaleMsFor(60_000)).toBe(SPAWN_LOCK_STALE_MS); + }); + + test("stretches past a long wait limit with describe headroom", () => { + expect(spawnLockStaleMsFor(15 * 60_000)).toBe(16 * 60_000); + }); +}); diff --git a/packages/shared/src/cli/commands/spawn-lock.ts b/packages/shared/src/cli/commands/spawn-lock.ts index d4f86c4a5..fe60ea24f 100644 --- a/packages/shared/src/cli/commands/spawn-lock.ts +++ b/packages/shared/src/cli/commands/spawn-lock.ts @@ -7,13 +7,30 @@ import path from "node:path"; * older than this the lock is presumed orphaned (the worker crashed/was killed * before it could release) and is stolen. * - * Must comfortably exceed the worker's worst-case runtime: the blocking preflight - * wait cap (PREFLIGHT_WAIT_MAX_MS = 5 min in the type-generator) plus a DESCRIBE - * budget. Six minutes leaves ~1 min of headroom over a worker that waits the full - * preflight window and then describes. + * Must comfortably exceed the worker's worst-case runtime: the default warehouse + * wait limit (5 min in the type-generator) plus a DESCRIBE budget. Six minutes + * leaves ~1 min of headroom over a worker that waits the full limit and then + * describes. A longer configured limit stretches it — see + * {@link spawnLockStaleMsFor}. */ export const SPAWN_LOCK_STALE_MS = 6 * 60 * 1000; +/** Headroom over the wait limit for the worker's DESCRIBE work. */ +const DESCRIBE_HEADROOM_MS = 60 * 1000; + +/** + * Staleness threshold for a worker that may wait up to `waitTimeoutMs` for the + * warehouse: never shorter than {@link SPAWN_LOCK_STALE_MS}, and always at + * least the limit plus describe headroom, so a live worker on a long limit + * isn't mistaken for an orphan. + */ +export function spawnLockStaleMsFor(waitTimeoutMs?: number): number { + if (waitTimeoutMs === undefined || !Number.isFinite(waitTimeoutMs)) { + return SPAWN_LOCK_STALE_MS; + } + return Math.max(SPAWN_LOCK_STALE_MS, waitTimeoutMs + DESCRIBE_HEADROOM_MS); +} + /** * Resolve the on-disk path of the single-flight spawn lock for a project. * diff --git a/packages/shared/src/cli/commands/type-generator.d.ts b/packages/shared/src/cli/commands/type-generator.d.ts index dfc9f3eb9..c18675a0f 100644 --- a/packages/shared/src/cli/commands/type-generator.d.ts +++ b/packages/shared/src/cli/commands/type-generator.d.ts @@ -22,6 +22,7 @@ declare module "@databricks/appkit/type-generator" { warehouseId: string; noCache?: boolean; mode?: "non-blocking" | "blocking"; + waitTimeoutMs?: number; }): Promise; export class TypegenSyntaxError extends Error {