From c0ef23ce6def39a347b98616b0b8010fd4136c49 Mon Sep 17 00:00:00 2001 From: Abdul-Kareem Aliu Date: Wed, 11 Nov 2026 22:48:18 +0000 Subject: [PATCH 1/6] air: Say when --idempotency-key returns an existing run Resubmitting `air run` with an already-used idempotency key (flag or YAML idempotency_token) printed "Submitted workload with Job Run ID" exactly like a fresh submit, so users could not tell a deduplicated submit from a new one (CNXT-2520). runs/submit returns only a run id, so when the token is caller-supplied the CLI now fetches the run and compares its command_path with the one this submit uploaded. Every submit uses a fresh launch directory, so a run pointing at a different command path was started by an earlier submit. Generated tokens skip the lookup. - Text output says "Returned existing workload with Job Run ID: (idempotency key "" was already used)". - JSON results (PENDING envelope, --watch SUBMITTED event and closing envelope) carry an `existing` boolean; dry runs omit it. - The fake workspace's runs/submit now honors idempotency tokens and keeps the submitted ai_runtime_task on the run. Co-authored-by: Isaac --- .nextchanges/air/idempotency-existing-run.md | 1 + acceptance/air/run-idempotency/databricks.yml | 2 + acceptance/air/run-idempotency/out.test.toml | 1 + acceptance/air/run-idempotency/output.txt | 52 +++++++++++++ acceptance/air/run-idempotency/run-token.yaml | 6 ++ acceptance/air/run-idempotency/run.yaml | 5 ++ acceptance/air/run-idempotency/script | 16 ++++ acceptance/air/run-idempotency/test.toml | 11 +++ cmd/air/logstream_support.go | 7 +- cmd/air/run.go | 29 ++++--- cmd/air/run_test.go | 78 +++++++++++++++++++ cmd/air/run_watch_test.go | 38 ++++++++- cmd/air/runsubmit.go | 66 ++++++++++++---- cmd/air/runsubmit_test.go | 30 +++---- cmd/air/telemetry_test.go | 2 +- libs/testserver/jobs.go | 16 +++- libs/testserver/jobs_test.go | 25 ++++++ 17 files changed, 338 insertions(+), 47 deletions(-) create mode 100644 .nextchanges/air/idempotency-existing-run.md create mode 100644 acceptance/air/run-idempotency/databricks.yml create mode 100644 acceptance/air/run-idempotency/out.test.toml create mode 100644 acceptance/air/run-idempotency/output.txt create mode 100644 acceptance/air/run-idempotency/run-token.yaml create mode 100644 acceptance/air/run-idempotency/run.yaml create mode 100644 acceptance/air/run-idempotency/script create mode 100644 acceptance/air/run-idempotency/test.toml diff --git a/.nextchanges/air/idempotency-existing-run.md b/.nextchanges/air/idempotency-existing-run.md new file mode 100644 index 00000000000..e8da4c7642b --- /dev/null +++ b/.nextchanges/air/idempotency-existing-run.md @@ -0,0 +1 @@ +* `air run` now says when `--idempotency-key` or `idempotency_token` returned an existing run, and its JSON output reports this as `existing`. diff --git a/acceptance/air/run-idempotency/databricks.yml b/acceptance/air/run-idempotency/databricks.yml new file mode 100644 index 00000000000..a1c79b6244a --- /dev/null +++ b/acceptance/air/run-idempotency/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: air-run-idempotency diff --git a/acceptance/air/run-idempotency/out.test.toml b/acceptance/air/run-idempotency/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/air/run-idempotency/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/air/run-idempotency/output.txt b/acceptance/air/run-idempotency/output.txt new file mode 100644 index 00000000000..d53421f4022 --- /dev/null +++ b/acceptance/air/run-idempotency/output.txt @@ -0,0 +1,52 @@ + +=== first submit with a key starts a run + +>>> [CLI] air run -f run.yaml --idempotency-key flag-key +Submitting experiment: idempotency-smoke +Submitted workload with Job Run ID: [FLAG_KEY_RUN_ID] +View job run at: [DATABRICKS_URL]/jobs/runs/[FLAG_KEY_RUN_ID] + +Tip: use --watch when submitting a run to stream logs to your terminal. +Stream logs after submission using: + databricks air logs [FLAG_KEY_RUN_ID] + +=== resubmitting with the same key returns the existing run + +>>> [CLI] air run -f run.yaml --idempotency-key flag-key +Submitting experiment: idempotency-smoke +Returned existing workload with Job Run ID: [FLAG_KEY_RUN_ID] (idempotency key "flag-key" was already used) +View job run at: [DATABRICKS_URL]/jobs/runs/[FLAG_KEY_RUN_ID] + +Tip: use --watch when submitting a run to stream logs to your terminal. +Stream logs after submission using: + databricks air logs [FLAG_KEY_RUN_ID] + +=== JSON output reports the existing run + +>>> [CLI] air run -f run.yaml --idempotency-key flag-key -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "status": "PENDING", + "run_id": "[FLAG_KEY_RUN_ID]", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/[FLAG_KEY_RUN_ID]", + "existing": true + } +} + +=== a different key starts a new run + +>>> [CLI] air run -f run.yaml --idempotency-key other-key -o json +false + +=== a YAML idempotency_token is deduplicated the same way + +>>> [CLI] air run -f run-token.yaml +Submitting experiment: idempotency-smoke +Returned existing workload with Job Run ID: [YAML_TOKEN_RUN_ID] (idempotency key "yaml-key" was already used) +View job run at: [DATABRICKS_URL]/jobs/runs/[YAML_TOKEN_RUN_ID] + +Tip: use --watch when submitting a run to stream logs to your terminal. +Stream logs after submission using: + databricks air logs [YAML_TOKEN_RUN_ID] diff --git a/acceptance/air/run-idempotency/run-token.yaml b/acceptance/air/run-idempotency/run-token.yaml new file mode 100644 index 00000000000..63dd8d7b961 --- /dev/null +++ b/acceptance/air/run-idempotency/run-token.yaml @@ -0,0 +1,6 @@ +experiment_name: idempotency-smoke +command: python train.py +compute: + accelerator_type: GPU_1xH100 + num_accelerators: 1 +idempotency_token: yaml-key diff --git a/acceptance/air/run-idempotency/run.yaml b/acceptance/air/run-idempotency/run.yaml new file mode 100644 index 00000000000..38eecf38446 --- /dev/null +++ b/acceptance/air/run-idempotency/run.yaml @@ -0,0 +1,5 @@ +experiment_name: idempotency-smoke +command: python train.py +compute: + accelerator_type: GPU_1xH100 + num_accelerators: 1 diff --git a/acceptance/air/run-idempotency/script b/acceptance/air/run-idempotency/script new file mode 100644 index 00000000000..146e3d8521b --- /dev/null +++ b/acceptance/air/run-idempotency/script @@ -0,0 +1,16 @@ +title "first submit with a key starts a run" +trace $CLI air run -f run.yaml --idempotency-key flag-key +add_repl.py "$($CLI air run -f run.yaml --idempotency-key flag-key -o json | jq -r .data.run_id)" FLAG_KEY_RUN_ID + +title "resubmitting with the same key returns the existing run" +trace $CLI air run -f run.yaml --idempotency-key flag-key + +title "JSON output reports the existing run" +trace $CLI air run -f run.yaml --idempotency-key flag-key -o json + +title "a different key starts a new run" +trace $CLI air run -f run.yaml --idempotency-key other-key -o json | jq '.data.existing' + +title "a YAML idempotency_token is deduplicated the same way" +add_repl.py "$($CLI air run -f run-token.yaml -o json | jq -r .data.run_id)" YAML_TOKEN_RUN_ID +trace $CLI air run -f run-token.yaml diff --git a/acceptance/air/run-idempotency/test.toml b/acceptance/air/run-idempotency/test.toml new file mode 100644 index 00000000000..266538a8a61 --- /dev/null +++ b/acceptance/air/run-idempotency/test.toml @@ -0,0 +1,11 @@ +# Repeated submits against the fake workspace, whose runs/submit returns the run +# an already-used idempotency token started. + +# The SDK probes host reachability with a HEAD request; stub it for determinism. +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +[[Server]] +Pattern = "POST /api/2.0/ai-training/config:validate" +Response.Body = '{}' diff --git a/cmd/air/logstream_support.go b/cmd/air/logstream_support.go index 69896f71c3c..1282e9cb419 100644 --- a/cmd/air/logstream_support.go +++ b/cmd/air/logstream_support.go @@ -127,15 +127,17 @@ type submittedEvent struct { TS string `json:"ts"` RunID string `json:"run_id"` DashboardURL string `json:"dashboard_url"` + Existing bool `json:"existing"` } // printSubmittedEvent writes the SUBMITTED JSONL event. -func printSubmittedEvent(out io.Writer, runID, dashboardURL string) { +func printSubmittedEvent(out io.Writer, runID, dashboardURL string, existing bool) { b, err := json.Marshal(submittedEvent{ Type: "SUBMITTED", TS: time.Now().UTC().Format(time.RFC3339), RunID: runID, DashboardURL: dashboardURL, + Existing: existing, }) if err != nil { return @@ -176,7 +178,7 @@ type terminalEvent struct { // printTerminalEvent writes the closing terminal-status envelope, matching the // shape of renderEnvelope(runResult). -func printTerminalEvent(out io.Writer, runID, status, dashboardURL string) { +func printTerminalEvent(out io.Writer, runID, status, dashboardURL string, existing bool) { b, err := json.Marshal(terminalEvent{ V: envelopeVersion, TS: time.Now().UTC().Format(time.RFC3339), @@ -184,6 +186,7 @@ func printTerminalEvent(out io.Writer, runID, status, dashboardURL string) { Status: status, RunID: runID, DashboardURL: dashboardURL, + Existing: &existing, }, }) if err != nil { diff --git a/cmd/air/run.go b/cmd/air/run.go index 332f1aa1218..9099aca9eeb 100644 --- a/cmd/air/run.go +++ b/cmd/air/run.go @@ -33,6 +33,8 @@ type runResult struct { DryRun bool `json:"dry_run,omitempty"` RunID string `json:"run_id,omitempty"` DashboardURL string `json:"dashboard_url,omitempty"` + // Existing is nil for a dry run, which submits nothing. + Existing *bool `json:"existing,omitempty"` } const experimentalContainersWarning = `+--------------------------------------------------------------------+ @@ -187,24 +189,28 @@ The path must be a separate argument: cobra reserves -h as a boolean, so } w := cmdctx.WorkspaceClient(ctx) - runID, dashboardURL, err := submitWorkload(ctx, w, cfg, file, idempotencyKey, !jsonOut) + runID, dashboardURL, existing, err := submitWorkload(ctx, w, cfg, file, idempotencyKey, !jsonOut) if err != nil { return err } + existingKey := "" + if existing { + existingKey = userIdempotencyToken(idempotencyKey, cfg) + } runIDStr := strconv.FormatInt(runID, 10) if !watch { if !jsonOut { out := cmd.OutOrStdout() - printSubmitResult(ctx, out, runIDStr, dashboardURL) + printSubmitResult(ctx, out, runIDStr, dashboardURL, existingKey) printPostSubmitGuidance(out, w.Config.Profile, runIDStr) grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, true) return nil } // PENDING is the submit status, distinct from the --watch JSONL // SUBMITTED event type below. - if err := renderEnvelope(ctx, runResult{Status: "PENDING", RunID: runIDStr, DashboardURL: dashboardURL}); err != nil { + if err := renderEnvelope(ctx, runResult{Status: "PENDING", RunID: runIDStr, DashboardURL: dashboardURL, Existing: &existing}); err != nil { return err } grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, false) @@ -238,7 +244,7 @@ The path must be a separate argument: cobra reserves -h as a boolean, so monitoringMessage = fmt.Sprintf("Monitoring run and streaming logs from node 0 of %d...", cfg.Compute.NumAccelerators/perNode) } // The MLflow links stream in via the logs below, so don't poll here. - printSubmitResult(ctx, out, runIDStr, dashboardURL) + printSubmitResult(ctx, out, runIDStr, dashboardURL, existingKey) grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, true) // Separate the submit summary from the streamed logs. fmt.Fprintln(out) @@ -251,7 +257,7 @@ The path must be a separate argument: cobra reserves -h as a boolean, so // STATUS events on each lifecycle transition, and a closing terminal-status // envelope after streaming. out := cmd.OutOrStdout() - printSubmittedEvent(out, runIDStr, dashboardURL) + printSubmittedEvent(out, runIDStr, dashboardURL, existing) grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, false) req.onStatusChange = func(current, previous string) { printStatusEvent(out, current, previous) @@ -262,7 +268,7 @@ The path must be a separate argument: cobra reserves -h as a boolean, so // the Bricklens path, so the terminal status must come from the run's // actual state — correct whether Bricklens or the MLflow fallback served // the logs. - printTerminalEvent(out, runIDStr, watchTerminalStatus(ctx, w, runID), dashboardURL) + printTerminalEvent(out, runIDStr, watchTerminalStatus(ctx, w, runID), dashboardURL, existing) return err } @@ -364,12 +370,17 @@ func handleWatchResult(out io.Writer, profile, runID string, err error) error { // printSubmitResult writes the green success line and Job Run link. The link is // styled (blue, underlined) and clickable, matching the `air get` view, and -// degrades to plain text on non-rich terminals. -func printSubmitResult(ctx context.Context, out io.Writer, runIDStr, dashboardURL string) { +// degrades to plain text on non-rich terminals. existingKey is the idempotency +// key when it returned an existing run, else empty. +func printSubmitResult(ctx context.Context, out io.Writer, runIDStr, dashboardURL, existingKey string) { renderer, colorOn := cmdio.NewRenderer(ctx, out) p := newPalette(renderer) - fmt.Fprintln(out, p.green.Render("Submitted workload with Job Run ID: "+runIDStr)) + line := "Submitted workload with Job Run ID: " + runIDStr + if existingKey != "" { + line = fmt.Sprintf("Returned existing workload with Job Run ID: %s (idempotency key %q was already used)", runIDStr, existingKey) + } + fmt.Fprintln(out, p.green.Render(line)) fmt.Fprintln(out, "View job run at: "+link(colorOn, p.blue, dashboardURL, dashboardURL)) } diff --git a/cmd/air/run_test.go b/cmd/air/run_test.go index 1e48b05a734..f26924f6319 100644 --- a/cmd/air/run_test.go +++ b/cmd/air/run_test.go @@ -128,10 +128,88 @@ func TestRunSubmitJSONStatusPending(t *testing.T) { out := buf.String() assert.Contains(t, out, `"status": "PENDING"`) assert.Contains(t, out, `"run_id": "555"`) + assert.Contains(t, out, `"existing": false`) // JSON stdout stays a clean envelope stream — no human-readable submit lines. assert.NotContains(t, out, "Submitting experiment") } +// idempotentSubmitServer runs the fake workspace, whose runs/submit returns the +// run a repeated idempotency token started. +func idempotentSubmitServer(t *testing.T) string { + t.Helper() + server := testserver.New(t) + t.Cleanup(server.Close) + server.Handle("POST", validateConfigPath, func(testserver.Request) any { + return validateConfigResponse{} + }) + testserver.AddDefaultHandlers(server) + return server.URL +} + +func runIdempotentSubmit(t *testing.T, srvURL string, out flags.Output, config string, args ...string) string { + t.Helper() + var buf bytes.Buffer + cfgPath := writeConfigFile(t, "run.yaml", config) + cmd := withOutput(newRunCommand(), out) + require.NoError(t, cmd.ParseFlags(append([]string{"--file", cfgPath}, args...))) + + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), out, nil, &buf, &buf, "", "")) + ctx = cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srvURL)) + cmd.SetContext(ctx) + cmd.SetOut(&buf) + require.NoError(t, cmd.RunE(cmd, nil)) + return buf.String() +} + +func TestRunSubmitIdempotencyKeyReturnsExistingRun(t *testing.T) { + srvURL := idempotentSubmitServer(t) + + first := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--idempotency-key", "flag-key") + assert.Contains(t, first, "Submitted workload with Job Run ID: ") + assert.NotContains(t, first, "existing") + + second := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--idempotency-key", "flag-key") + assert.Contains(t, second, "Returned existing workload with Job Run ID: ") + assert.Contains(t, second, `(idempotency key "flag-key" was already used)`) + assert.NotContains(t, second, "Submitted workload") + assert.Contains(t, second, "View job run at: ") + + other := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--idempotency-key", "other-key") + assert.Contains(t, other, "Submitted workload with Job Run ID: ") +} + +func TestRunSubmitYAMLIdempotencyTokenReturnsExistingRun(t *testing.T) { + srvURL := idempotentSubmitServer(t) + config := minimalConfig + "idempotency_token: yaml-key\n" + + first := runIdempotentSubmit(t, srvURL, flags.OutputText, config) + assert.Contains(t, first, "Submitted workload with Job Run ID: ") + + second := runIdempotentSubmit(t, srvURL, flags.OutputText, config) + assert.Contains(t, second, `(idempotency key "yaml-key" was already used)`) + assert.NotContains(t, second, "Submitted workload") +} + +func TestRunSubmitJSONReportsExistingRun(t *testing.T) { + srvURL := idempotentSubmitServer(t) + decode := func(out string) runResult { + var envelope struct { + Data runResult `json:"data"` + } + require.NoError(t, json.Unmarshal([]byte(out), &envelope), out) + require.NotNil(t, envelope.Data.Existing) + return envelope.Data + } + + first := decode(runIdempotentSubmit(t, srvURL, flags.OutputJSON, minimalConfig, "--idempotency-key", "json-key")) + assert.False(t, *first.Existing) + + second := decode(runIdempotentSubmit(t, srvURL, flags.OutputJSON, minimalConfig, "--idempotency-key", "json-key")) + assert.True(t, *second.Existing) + assert.Equal(t, first.RunID, second.RunID) + assert.Equal(t, "PENDING", second.Status) +} + type dryRunRequest struct { method string path string diff --git a/cmd/air/run_watch_test.go b/cmd/air/run_watch_test.go index 52a859de21b..ed29343f5c8 100644 --- a/cmd/air/run_watch_test.go +++ b/cmd/air/run_watch_test.go @@ -21,6 +21,8 @@ import ( // watchServer serves submit, the auth probe, a terminal runs/get, and a single // page of Bricklens logs — everything `air run --watch` touches after submit. // resultState is the terminal result the run reports (e.g. SUCCESS, FAILED). +// The run's command path is from an earlier submit's launch directory, so a +// submit with an idempotency key sees it as an existing run. func watchServer(t *testing.T, resultState string) *httptest.Server { t.Helper() runGet := `{ @@ -28,7 +30,11 @@ func watchServer(t *testing.T, resultState string) *httptest.Server { "start_time": 1700000000000, "end_time": 1700000012000, "state": {"life_cycle_state": "TERMINATED", "result_state": "` + resultState + `"}, - "tasks": [{"run_id": 778, "attempt_number": 0}] + "tasks": [{ + "run_id": 778, + "attempt_number": 0, + "ai_runtime_task": {"deployments": [{"command_path": "/Users/u@example.com/.air/cli_launch/my-run/my-run_0123456789abcdef/command.sh"}]} + }] }` srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { @@ -139,6 +145,34 @@ func TestRunWatchStreamsLogs(t *testing.T) { assert.Contains(t, out, "step 1\nstep 2") } +const idempotentWatchConfig = minimalConfig + "idempotency_token: watch-key\n" + +func TestRunWatchExistingRunStreamsLogs(t *testing.T) { + var buf bytes.Buffer + err := runWatchCmdWithConfig(t, flags.OutputText, &buf, watchServer(t, "SUCCESS").URL, idempotentWatchConfig) + require.NoError(t, err) + + out := buf.String() + assert.Contains(t, out, `Returned existing workload with Job Run ID: 777 (idempotency key "watch-key" was already used)`) + assert.NotContains(t, out, "Submitted workload") + assert.Contains(t, out, "View job run at: ") + assert.Contains(t, out, "step 1\nstep 2") +} + +func TestRunWatchJSONExistingRun(t *testing.T) { + var buf bytes.Buffer + err := runWatchCmdWithConfig(t, flags.OutputJSON, &buf, watchServer(t, "SUCCESS").URL, idempotentWatchConfig) + require.NoError(t, err) + + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + require.GreaterOrEqual(t, len(lines), 2) + assert.Contains(t, lines[0], `"type":"SUBMITTED"`) + assert.Contains(t, lines[0], `"existing":true`) + assert.Contains(t, buf.String(), `"line":"step 1"`) + assert.Contains(t, lines[len(lines)-1], `"status":"SUCCESS"`) + assert.Contains(t, lines[len(lines)-1], `"existing":true`) +} + func TestRunWatchMultinodeIdentifiesStreamedNode(t *testing.T) { var buf bytes.Buffer err := runWatchCmdWithConfig(t, flags.OutputText, &buf, watchServer(t, "SUCCESS").URL, multinodeWatchConfig) @@ -158,6 +192,7 @@ func TestRunWatchJSONEmitsSubmittedThenLogs(t *testing.T) { // First event is SUBMITTED with the run id; then STATUS + streamed LOG events. assert.Contains(t, lines[0], `"type":"SUBMITTED"`) assert.Contains(t, lines[0], `"run_id":"777"`) + assert.Contains(t, lines[0], `"existing":false`) assert.Contains(t, all, `"type":"STATUS"`) assert.Contains(t, all, `"type":"LOG"`) assert.Contains(t, all, `"line":"step 1"`) @@ -165,6 +200,7 @@ func TestRunWatchJSONEmitsSubmittedThenLogs(t *testing.T) { // The last line is the closing terminal-status envelope carrying SUCCESS. assert.Contains(t, lines[len(lines)-1], `"status":"SUCCESS"`) assert.Contains(t, lines[len(lines)-1], `"run_id":"777"`) + assert.Contains(t, lines[len(lines)-1], `"existing":false`) } func TestRunWatchJSONFailedRunTerminalEnvelope(t *testing.T) { diff --git a/cmd/air/runsubmit.go b/cmd/air/runsubmit.go index f01198d2b7a..c28b8b1d973 100644 --- a/cmd/air/runsubmit.go +++ b/cmd/air/runsubmit.go @@ -17,6 +17,7 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/compute" @@ -267,19 +268,43 @@ func aiRuntimeTaskFromSubmitBody(body map[string]any) (map[string]any, error) { return aiRuntimeTask, nil } +// userIdempotencyToken returns the caller-supplied token: the --idempotency-key +// flag wins over the config's token. It is empty when neither is set. +func userIdempotencyToken(flag string, cfg *runConfig) string { + if flag == "" && cfg.IdempotencyToken != nil { + return *cfg.IdempotencyToken + } + return flag +} + // effectiveIdempotencyToken resolves the token sent to validation and submit: -// the --idempotency-key flag wins, then the config's token, else a generated one. +// the caller-supplied token, else a generated one. func effectiveIdempotencyToken(flag string, cfg *runConfig) string { - token := flag - if token == "" && cfg.IdempotencyToken != nil { - token = *cfg.IdempotencyToken - } + token := userIdempotencyToken(flag, cfg) if token == "" { token = uuid.New().String() } return token } +// isExistingRun reports whether runID was started by an earlier submit that +// used the same idempotency token. runs/submit returns only a run id, so the +// run's command path is the signal: every submit uploads to a fresh launch +// directory, and a fresh run references this submit's commandPath while an +// existing run references an earlier one's. +func isExistingRun(ctx context.Context, w *databricks.WorkspaceClient, runID int64, commandPath string) bool { + run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: runID}) + if err != nil { + log.Warnf(ctx, "could not check whether run %d already existed: %v", runID, err) + return false + } + task := aiRuntimeTaskOf(run) + if task == nil || len(task.Deployments) == 0 || task.Deployments[0].CommandPath == "" { + return false + } + return task.Deployments[0].CommandPath != commandPath +} + // validateIdempotencyToken keeps the CLI's submission guard as a fallback when // backend validation is unavailable or does not yet enforce the same rule. func validateIdempotencyToken(token string) error { @@ -330,9 +355,10 @@ func stageRunArtifacts(ctx context.Context, launchWriter fileWriter, items []upl // submitWorkload runs the submit happy path: validate the config while uploading // the launch artifacts, ensure the experiment directory, assemble the Jobs // payload, and submit it. It -// returns the new run_id and its dashboard URL. showProgress enables the stderr -// staging spinner (text mode only). -func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig, configPath, idempotencyKey string, showProgress bool) (runID int64, dashboardURL string, err error) { +// returns the run_id and its dashboard URL, and whether a caller-supplied +// idempotency token returned an existing run instead of starting a new one. +// showProgress enables the stderr staging spinner (text mode only). +func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig, configPath, idempotencyKey string, showProgress bool) (runID int64, dashboardURL string, existing bool, err error) { start := time.Now() var snap snapshotResult defer func() { logRunEvent(ctx, cfg, snap, runID, time.Since(start), err) }() @@ -340,7 +366,7 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run // Compute and validate the actual submission path before creating artifacts. base, funcDir, commandPath, err := prospectiveLaunchPaths(ctx, w, cfg) if err != nil { - return 0, "", err + return 0, "", false, err } containers := submittedContainers(cfg, funcDir) token := effectiveIdempotencyToken(idempotencyKey, cfg) @@ -349,11 +375,11 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run fc, err := filer.NewWorkspaceFilesClient(w, funcDir) if err != nil { - return 0, "", err + return 0, "", false, err } items, err := buildArtifacts(cfg) if err != nil { - return 0, "", err + return 0, "", false, err } // Package and upload the code snapshot, if any; the remote code_source_path @@ -388,7 +414,7 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run }) }) if err := group.Wait(); err != nil { - return 0, "", err + return 0, "", false, err } // Warn only after staging so the warning doesn't interleave with the spinner. if validationUnavailable && cmdio.HasIO(ctx) { @@ -398,7 +424,7 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run // Retain the local guard after backend validation so an unavailable endpoint // cannot let a bad key reach submission. if err := validateIdempotencyToken(token); err != nil { - return 0, "", err + return 0, "", false, err } // Resolve the usage policy to its id next, so a bad name fails with a clear @@ -412,7 +438,7 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run if cfg.UsagePolicyName != nil { usagePolicyID, err = resolveUsagePolicyIDByName(ctx, w, *cfg.UsagePolicyName) if err != nil { - return 0, "", err + return 0, "", false, err } } @@ -421,7 +447,7 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run experimentDir = *cfg.MLflowExperimentDirectory } if err := ensureExperimentDirectory(ctx, w, experimentDir); err != nil { - return 0, "", err + return 0, "", false, err } runtimeVersion, _ := cfg.runtimeVersion() @@ -443,11 +469,17 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run // the best-effort grant never delays the success line. runID, err = submitRun(ctx, w, payload, poolID, priorityClass, cfg.unityCatalogImagePath(), containers) if err != nil { - return 0, "", err + return 0, "", false, err + } + + // A generated token is never a repeat. Container runs carry their command + // paths on containers, which the SDK's DeploymentSpec does not model. + if userIdempotencyToken(idempotencyKey, cfg) != "" && len(containers) == 0 { + existing = isExistingRun(ctx, w, runID, commandPath) } dashboardURL = strings.TrimRight(w.Config.Host, "/") + "/jobs/runs/" + strconv.FormatInt(runID, 10) - return runID, dashboardURL, nil + return runID, dashboardURL, existing, nil } func submittedContainers(cfg *runConfig, funcDir string) []submittedContainer { diff --git a/cmd/air/runsubmit_test.go b/cmd/air/runsubmit_test.go index 441a1c12d26..ff5128c2f04 100644 --- a/cmd/air/runsubmit_test.go +++ b/cmd/air/runsubmit_test.go @@ -379,7 +379,7 @@ func TestSubmitWorkload(t *testing.T) { cfg, err := loadRunConfig(cfgPath) require.NoError(t, err) - runID, dashboardURL, err := submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) + runID, dashboardURL, _, err := submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) require.NoError(t, err) assert.Equal(t, int64(777), runID) assert.Contains(t, dashboardURL, "/jobs/runs/777") @@ -448,7 +448,7 @@ func TestSubmitWorkloadValidationOutcome(t *testing.T) { require.NoError(t, err) ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - _, _, err = submitWorkload(ctx, w, cfg, cfgPath, "idem-key", false) + _, _, _, err = submitWorkload(ctx, w, cfg, cfgPath, "idem-key", false) if tt.wantError != "" { require.ErrorContains(t, err, tt.wantError) } else { @@ -499,7 +499,7 @@ func TestSubmitWorkloadValidatesWhileStaging(t *testing.T) { cfg, err := loadRunConfig(cfgPath) require.NoError(t, err) - runID, _, err := submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) + runID, _, _, err := submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) require.NoError(t, err) assert.Equal(t, int64(777), runID) } @@ -535,7 +535,7 @@ func TestSubmitWorkloadValidationFailureCancelsStaging(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() - _, _, err = submitWorkload(ctx, w, cfg, cfgPath, "idem-key", false) + _, _, _, err = submitWorkload(ctx, w, cfg, cfgPath, "idem-key", false) require.ErrorContains(t, err, "task.experiment: is invalid") require.NoError(t, ctx.Err(), "staging was not cancelled by the validation failure") assert.Zero(t, submissions.Load()) @@ -562,7 +562,7 @@ func TestSubmitWorkloadStagingErrorPreventsSubmit(t *testing.T) { cfg, err := loadRunConfig(cfgPath) require.NoError(t, err) - _, _, err = submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) + _, _, _, err = submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) require.Error(t, err) assert.ErrorContains(t, err, "failed to create launch directory") assert.Zero(t, submitCalls.Load()) @@ -592,7 +592,7 @@ func TestSubmitWorkloadHonorsOverride(t *testing.T) { cfg, err := loadRunConfigWithOverrides(t.Context(), cfgPath, []string{"compute.num_accelerators=4"}) require.NoError(t, err) - _, _, err = submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) + _, _, _, err = submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) require.NoError(t, err) require.Len(t, got.Tasks, 1) @@ -623,7 +623,7 @@ environment: cfg, err := loadRunConfig(cfgPath) require.NoError(t, err) - _, _, err = submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) + _, _, _, err = submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) require.NoError(t, err) tasks, ok := got["tasks"].([]any) @@ -696,7 +696,7 @@ containers: `) cfg, err := loadRunConfig(cfgPath) require.NoError(t, err) - _, _, err = submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) + _, _, _, err = submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) require.NoError(t, err) aiRuntimeTask, err := aiRuntimeTaskFromSubmitBody(got) @@ -752,7 +752,7 @@ code_source: // The upload path logs via cmdio; the real `air run` context carries it. ctx := cmdio.MockDiscard(t.Context()) - _, _, err = submitWorkload(ctx, w, loaded, cfgPath, "idem", false) + _, _, _, err = submitWorkload(ctx, w, loaded, cfgPath, "idem", false) require.NoError(t, err) at := got.Tasks[0].AiRuntimeTask @@ -797,7 +797,7 @@ code_source: require.NoError(t, err) ctx := cmdio.MockDiscard(t.Context()) - _, _, err = submitWorkload(ctx, w, loaded, cfgPath, "idem", false) + _, _, _, err = submitWorkload(ctx, w, loaded, cfgPath, "idem", false) require.NoError(t, err) at := got.Tasks[0].AiRuntimeTask @@ -1040,7 +1040,7 @@ code_source: require.NoError(t, err) ctx := cmdio.MockDiscard(t.Context()) - _, _, err = submitWorkload(ctx, w, loaded, cfgPath, "idem", false) + _, _, _, err = submitWorkload(ctx, w, loaded, cfgPath, "idem", false) require.NoError(t, err) at := got.Tasks[0].AiRuntimeTask @@ -1072,7 +1072,7 @@ func TestSubmitWorkloadGuards(t *testing.T) { cfg := *base cfg.UsagePolicyName = new("nope") - _, _, err = submitWorkload(t.Context(), pw, &cfg, cfgPath, "", false) + _, _, _, err = submitWorkload(t.Context(), pw, &cfg, cfgPath, "", false) require.ErrorContains(t, err, `no usage policy named "nope"`) assert.Zero(t, submissions.Load()) }) @@ -1094,7 +1094,7 @@ func TestSubmitWorkloadGuards(t *testing.T) { cfg := *base cfg.UsagePolicyName = new("nope") - _, _, err = submitWorkload(t.Context(), pw, &cfg, cfgPath, "", false) + _, _, _, err = submitWorkload(t.Context(), pw, &cfg, cfgPath, "", false) require.ErrorContains(t, err, "task.experiment: is invalid") assert.Zero(t, policyLookups.Load()) }) @@ -1129,7 +1129,7 @@ func TestSubmitWorkloadSendsUsagePolicy(t *testing.T) { cfg, err := loadRunConfig(cfgPath) require.NoError(t, err) - _, _, err = submitWorkload(cmdio.MockDiscard(t.Context()), w, cfg, cfgPath, "idem", false) + _, _, _, err = submitWorkload(cmdio.MockDiscard(t.Context()), w, cfg, cfgPath, "idem", false) require.NoError(t, err) assert.Equal(t, policyID, got.BudgetPolicyId) }) @@ -1140,7 +1140,7 @@ func TestSubmitWorkloadSendsUsagePolicy(t *testing.T) { cfg, err := loadRunConfig(cfgPath) require.NoError(t, err) - _, _, err = submitWorkload(cmdio.MockDiscard(t.Context()), w, cfg, cfgPath, "idem", false) + _, _, _, err = submitWorkload(cmdio.MockDiscard(t.Context()), w, cfg, cfgPath, "idem", false) require.NoError(t, err) assert.Equal(t, policyID, got.BudgetPolicyId) }) diff --git a/cmd/air/telemetry_test.go b/cmd/air/telemetry_test.go index 35a0cad984d..d57e59c09b7 100644 --- a/cmd/air/telemetry_test.go +++ b/cmd/air/telemetry_test.go @@ -89,7 +89,7 @@ func TestAirRunTelemetry(t *testing.T) { cfgPath := writeConfigFile(t, "run.yaml", configYAML) cfg, err := loadRunConfig(cfgPath) require.NoError(t, err) - _, _, err = submitWorkload(ctx, w, cfg, cfgPath, "idem", false) + _, _, _, err = submitWorkload(ctx, w, cfg, cfgPath, "idem", false) if tc.fail || tc.failUpload { require.Error(t, err) } else { diff --git a/libs/testserver/jobs.go b/libs/testserver/jobs.go index 772d868cb14..8c4669c945f 100644 --- a/libs/testserver/jobs.go +++ b/libs/testserver/jobs.go @@ -647,6 +647,13 @@ func (s *FakeWorkspace) JobsSubmit(req Request) Response { defer s.LockUnlock()() + // A resent request with the same non-empty token returns the run that token started. + if request.IdempotencyToken != "" { + if runId, ok := s.JobRunIdempotency[request.IdempotencyToken]; ok { + return Response{Body: jobs.SubmitRunResponse{RunId: runId}} + } + } + runId := nextID() // The default run name for one-time runs is "Untitled" (Jobs API behavior). @@ -659,8 +666,9 @@ func (s *FakeWorkspace) JobsSubmit(req Request) Response { var tasks []jobs.RunTask for _, t := range request.Tasks { tasks = append(tasks, jobs.RunTask{ - RunId: nextID(), - TaskKey: t.TaskKey, + RunId: nextID(), + TaskKey: t.TaskKey, + AiRuntimeTask: t.AiRuntimeTask, State: &jobs.RunState{ LifeCycleState: jobs.RunLifeCycleStateRunning, }, @@ -679,6 +687,10 @@ func (s *FakeWorkspace) JobsSubmit(req Request) Response { Tasks: tasks, } + if request.IdempotencyToken != "" { + s.JobRunIdempotency[request.IdempotencyToken] = runId + } + // No tunnel server runs locally, so synthesize the metadata.json it would // publish; `ssh connect` polls for it before connecting. if strings.HasPrefix(runName, sshTunnelBootstrapRunPrefix) { diff --git a/libs/testserver/jobs_test.go b/libs/testserver/jobs_test.go index a5b04542deb..8228b6a7679 100644 --- a/libs/testserver/jobs_test.go +++ b/libs/testserver/jobs_test.go @@ -72,6 +72,31 @@ func TestJobsSubmit_DefaultsRunNameToUntitled(t *testing.T) { assert.Equal(t, "Untitled", run.RunName) } +func TestJobsSubmit_IdempotencyTokenReturnsExistingRun(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + task := func(commandPath string) []jobs.SubmitTask { + return []jobs.SubmitTask{{ + TaskKey: "main", + AiRuntimeTask: &jobs.AiRuntimeTask{ + Deployments: []jobs.DeploymentSpec{{CommandPath: commandPath}}, + }, + }} + } + + first := submitRun(t, workspace, jobs.SubmitRun{IdempotencyToken: "token", Tasks: task("/first/command.sh")}) + second := submitRun(t, workspace, jobs.SubmitRun{IdempotencyToken: "token", Tasks: task("/second/command.sh")}) + untokened := submitRun(t, workspace, jobs.SubmitRun{Tasks: task("/third/command.sh")}) + + assert.Equal(t, first.RunId, second.RunId) + assert.NotEqual(t, first.RunId, untokened.RunId) + + // The resent request must not overwrite the original run's spec. + run := getRun(t, workspace, first.RunId) + require.Len(t, run.Tasks, 1) + require.NotNil(t, run.Tasks[0].AiRuntimeTask) + assert.Equal(t, "/first/command.sh", run.Tasks[0].AiRuntimeTask.Deployments[0].CommandPath) +} + func TestJobsSubmit_RunReachesTerminalStateOnPoll(t *testing.T) { workspace := NewFakeWorkspace("http://test", "dbapi123") From 10ffafb8b9de57d13860789265afa2045200d594 Mon Sep 17 00:00:00 2001 From: Abdul-Kareem Aliu Date: Wed, 11 Nov 2026 23:55:39 +0000 Subject: [PATCH 2/6] Add PR link to changelog entry Co-authored-by: Isaac --- .nextchanges/air/idempotency-existing-run.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/air/idempotency-existing-run.md b/.nextchanges/air/idempotency-existing-run.md index e8da4c7642b..5f854e8d002 100644 --- a/.nextchanges/air/idempotency-existing-run.md +++ b/.nextchanges/air/idempotency-existing-run.md @@ -1 +1 @@ -* `air run` now says when `--idempotency-key` or `idempotency_token` returned an existing run, and its JSON output reports this as `existing`. +* `air run` now says when `--idempotency-key` or `idempotency_token` returned an existing run, and its JSON output reports this as `existing`. ([#7527](https://github.com/databricks/cli/pull/7527)) From 23091166d2dae14be4b6278a057b2a4a5c853991 Mon Sep 17 00:00:00 2001 From: Abdul-Kareem Aliu Date: Thu, 12 Nov 2026 00:04:38 +0000 Subject: [PATCH 3/6] air: Narrow existing-run reporting to text and JSON result Keep submitWorkload's signature unchanged: run opts in through the context, and submitWorkload records whether a caller-supplied idempotency token returned an existing run. Drop `existing` from the --watch JSONL events; the text line and the non-watch -o json result still report it. Co-authored-by: Isaac --- cmd/air/logstream_support.go | 7 ++----- cmd/air/run.go | 11 +++++----- cmd/air/run_watch_test.go | 38 +---------------------------------- cmd/air/runsubmit.go | 39 ++++++++++++++++++++++-------------- cmd/air/runsubmit_test.go | 30 +++++++++++++-------------- cmd/air/telemetry_test.go | 2 +- 6 files changed, 49 insertions(+), 78 deletions(-) diff --git a/cmd/air/logstream_support.go b/cmd/air/logstream_support.go index 1282e9cb419..69896f71c3c 100644 --- a/cmd/air/logstream_support.go +++ b/cmd/air/logstream_support.go @@ -127,17 +127,15 @@ type submittedEvent struct { TS string `json:"ts"` RunID string `json:"run_id"` DashboardURL string `json:"dashboard_url"` - Existing bool `json:"existing"` } // printSubmittedEvent writes the SUBMITTED JSONL event. -func printSubmittedEvent(out io.Writer, runID, dashboardURL string, existing bool) { +func printSubmittedEvent(out io.Writer, runID, dashboardURL string) { b, err := json.Marshal(submittedEvent{ Type: "SUBMITTED", TS: time.Now().UTC().Format(time.RFC3339), RunID: runID, DashboardURL: dashboardURL, - Existing: existing, }) if err != nil { return @@ -178,7 +176,7 @@ type terminalEvent struct { // printTerminalEvent writes the closing terminal-status envelope, matching the // shape of renderEnvelope(runResult). -func printTerminalEvent(out io.Writer, runID, status, dashboardURL string, existing bool) { +func printTerminalEvent(out io.Writer, runID, status, dashboardURL string) { b, err := json.Marshal(terminalEvent{ V: envelopeVersion, TS: time.Now().UTC().Format(time.RFC3339), @@ -186,7 +184,6 @@ func printTerminalEvent(out io.Writer, runID, status, dashboardURL string, exist Status: status, RunID: runID, DashboardURL: dashboardURL, - Existing: &existing, }, }) if err != nil { diff --git a/cmd/air/run.go b/cmd/air/run.go index 9099aca9eeb..9009a2a02e9 100644 --- a/cmd/air/run.go +++ b/cmd/air/run.go @@ -189,12 +189,13 @@ The path must be a separate argument: cobra reserves -h as a boolean, so } w := cmdctx.WorkspaceClient(ctx) - runID, dashboardURL, existing, err := submitWorkload(ctx, w, cfg, file, idempotencyKey, !jsonOut) + submitCtx, existing := withExistingRunReport(ctx) + runID, dashboardURL, err := submitWorkload(submitCtx, w, cfg, file, idempotencyKey, !jsonOut) if err != nil { return err } existingKey := "" - if existing { + if *existing { existingKey = userIdempotencyToken(idempotencyKey, cfg) } @@ -210,7 +211,7 @@ The path must be a separate argument: cobra reserves -h as a boolean, so } // PENDING is the submit status, distinct from the --watch JSONL // SUBMITTED event type below. - if err := renderEnvelope(ctx, runResult{Status: "PENDING", RunID: runIDStr, DashboardURL: dashboardURL, Existing: &existing}); err != nil { + if err := renderEnvelope(ctx, runResult{Status: "PENDING", RunID: runIDStr, DashboardURL: dashboardURL, Existing: existing}); err != nil { return err } grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, false) @@ -257,7 +258,7 @@ The path must be a separate argument: cobra reserves -h as a boolean, so // STATUS events on each lifecycle transition, and a closing terminal-status // envelope after streaming. out := cmd.OutOrStdout() - printSubmittedEvent(out, runIDStr, dashboardURL, existing) + printSubmittedEvent(out, runIDStr, dashboardURL) grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, false) req.onStatusChange = func(current, previous string) { printStatusEvent(out, current, previous) @@ -268,7 +269,7 @@ The path must be a separate argument: cobra reserves -h as a boolean, so // the Bricklens path, so the terminal status must come from the run's // actual state — correct whether Bricklens or the MLflow fallback served // the logs. - printTerminalEvent(out, runIDStr, watchTerminalStatus(ctx, w, runID), dashboardURL, existing) + printTerminalEvent(out, runIDStr, watchTerminalStatus(ctx, w, runID), dashboardURL) return err } diff --git a/cmd/air/run_watch_test.go b/cmd/air/run_watch_test.go index ed29343f5c8..52a859de21b 100644 --- a/cmd/air/run_watch_test.go +++ b/cmd/air/run_watch_test.go @@ -21,8 +21,6 @@ import ( // watchServer serves submit, the auth probe, a terminal runs/get, and a single // page of Bricklens logs — everything `air run --watch` touches after submit. // resultState is the terminal result the run reports (e.g. SUCCESS, FAILED). -// The run's command path is from an earlier submit's launch directory, so a -// submit with an idempotency key sees it as an existing run. func watchServer(t *testing.T, resultState string) *httptest.Server { t.Helper() runGet := `{ @@ -30,11 +28,7 @@ func watchServer(t *testing.T, resultState string) *httptest.Server { "start_time": 1700000000000, "end_time": 1700000012000, "state": {"life_cycle_state": "TERMINATED", "result_state": "` + resultState + `"}, - "tasks": [{ - "run_id": 778, - "attempt_number": 0, - "ai_runtime_task": {"deployments": [{"command_path": "/Users/u@example.com/.air/cli_launch/my-run/my-run_0123456789abcdef/command.sh"}]} - }] + "tasks": [{"run_id": 778, "attempt_number": 0}] }` srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { @@ -145,34 +139,6 @@ func TestRunWatchStreamsLogs(t *testing.T) { assert.Contains(t, out, "step 1\nstep 2") } -const idempotentWatchConfig = minimalConfig + "idempotency_token: watch-key\n" - -func TestRunWatchExistingRunStreamsLogs(t *testing.T) { - var buf bytes.Buffer - err := runWatchCmdWithConfig(t, flags.OutputText, &buf, watchServer(t, "SUCCESS").URL, idempotentWatchConfig) - require.NoError(t, err) - - out := buf.String() - assert.Contains(t, out, `Returned existing workload with Job Run ID: 777 (idempotency key "watch-key" was already used)`) - assert.NotContains(t, out, "Submitted workload") - assert.Contains(t, out, "View job run at: ") - assert.Contains(t, out, "step 1\nstep 2") -} - -func TestRunWatchJSONExistingRun(t *testing.T) { - var buf bytes.Buffer - err := runWatchCmdWithConfig(t, flags.OutputJSON, &buf, watchServer(t, "SUCCESS").URL, idempotentWatchConfig) - require.NoError(t, err) - - lines := strings.Split(strings.TrimSpace(buf.String()), "\n") - require.GreaterOrEqual(t, len(lines), 2) - assert.Contains(t, lines[0], `"type":"SUBMITTED"`) - assert.Contains(t, lines[0], `"existing":true`) - assert.Contains(t, buf.String(), `"line":"step 1"`) - assert.Contains(t, lines[len(lines)-1], `"status":"SUCCESS"`) - assert.Contains(t, lines[len(lines)-1], `"existing":true`) -} - func TestRunWatchMultinodeIdentifiesStreamedNode(t *testing.T) { var buf bytes.Buffer err := runWatchCmdWithConfig(t, flags.OutputText, &buf, watchServer(t, "SUCCESS").URL, multinodeWatchConfig) @@ -192,7 +158,6 @@ func TestRunWatchJSONEmitsSubmittedThenLogs(t *testing.T) { // First event is SUBMITTED with the run id; then STATUS + streamed LOG events. assert.Contains(t, lines[0], `"type":"SUBMITTED"`) assert.Contains(t, lines[0], `"run_id":"777"`) - assert.Contains(t, lines[0], `"existing":false`) assert.Contains(t, all, `"type":"STATUS"`) assert.Contains(t, all, `"type":"LOG"`) assert.Contains(t, all, `"line":"step 1"`) @@ -200,7 +165,6 @@ func TestRunWatchJSONEmitsSubmittedThenLogs(t *testing.T) { // The last line is the closing terminal-status envelope carrying SUCCESS. assert.Contains(t, lines[len(lines)-1], `"status":"SUCCESS"`) assert.Contains(t, lines[len(lines)-1], `"run_id":"777"`) - assert.Contains(t, lines[len(lines)-1], `"existing":false`) } func TestRunWatchJSONFailedRunTerminalEnvelope(t *testing.T) { diff --git a/cmd/air/runsubmit.go b/cmd/air/runsubmit.go index c28b8b1d973..04bd5720ad6 100644 --- a/cmd/air/runsubmit.go +++ b/cmd/air/runsubmit.go @@ -287,6 +287,16 @@ func effectiveIdempotencyToken(flag string, cfg *runConfig) string { return token } +type existingRunKey struct{} + +// withExistingRunReport returns a context in which submitWorkload sets the +// returned bool when a caller-supplied idempotency token returned an existing +// run. Only the run command opts in, as the check costs an extra runs/get. +func withExistingRunReport(ctx context.Context) (context.Context, *bool) { + existing := new(bool) + return context.WithValue(ctx, existingRunKey{}, existing), existing +} + // isExistingRun reports whether runID was started by an earlier submit that // used the same idempotency token. runs/submit returns only a run id, so the // run's command path is the signal: every submit uploads to a fresh launch @@ -355,10 +365,9 @@ func stageRunArtifacts(ctx context.Context, launchWriter fileWriter, items []upl // submitWorkload runs the submit happy path: validate the config while uploading // the launch artifacts, ensure the experiment directory, assemble the Jobs // payload, and submit it. It -// returns the run_id and its dashboard URL, and whether a caller-supplied -// idempotency token returned an existing run instead of starting a new one. -// showProgress enables the stderr staging spinner (text mode only). -func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig, configPath, idempotencyKey string, showProgress bool) (runID int64, dashboardURL string, existing bool, err error) { +// returns the new run_id and its dashboard URL. showProgress enables the stderr +// staging spinner (text mode only). +func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig, configPath, idempotencyKey string, showProgress bool) (runID int64, dashboardURL string, err error) { start := time.Now() var snap snapshotResult defer func() { logRunEvent(ctx, cfg, snap, runID, time.Since(start), err) }() @@ -366,7 +375,7 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run // Compute and validate the actual submission path before creating artifacts. base, funcDir, commandPath, err := prospectiveLaunchPaths(ctx, w, cfg) if err != nil { - return 0, "", false, err + return 0, "", err } containers := submittedContainers(cfg, funcDir) token := effectiveIdempotencyToken(idempotencyKey, cfg) @@ -375,11 +384,11 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run fc, err := filer.NewWorkspaceFilesClient(w, funcDir) if err != nil { - return 0, "", false, err + return 0, "", err } items, err := buildArtifacts(cfg) if err != nil { - return 0, "", false, err + return 0, "", err } // Package and upload the code snapshot, if any; the remote code_source_path @@ -414,7 +423,7 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run }) }) if err := group.Wait(); err != nil { - return 0, "", false, err + return 0, "", err } // Warn only after staging so the warning doesn't interleave with the spinner. if validationUnavailable && cmdio.HasIO(ctx) { @@ -424,7 +433,7 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run // Retain the local guard after backend validation so an unavailable endpoint // cannot let a bad key reach submission. if err := validateIdempotencyToken(token); err != nil { - return 0, "", false, err + return 0, "", err } // Resolve the usage policy to its id next, so a bad name fails with a clear @@ -438,7 +447,7 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run if cfg.UsagePolicyName != nil { usagePolicyID, err = resolveUsagePolicyIDByName(ctx, w, *cfg.UsagePolicyName) if err != nil { - return 0, "", false, err + return 0, "", err } } @@ -447,7 +456,7 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run experimentDir = *cfg.MLflowExperimentDirectory } if err := ensureExperimentDirectory(ctx, w, experimentDir); err != nil { - return 0, "", false, err + return 0, "", err } runtimeVersion, _ := cfg.runtimeVersion() @@ -469,17 +478,17 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run // the best-effort grant never delays the success line. runID, err = submitRun(ctx, w, payload, poolID, priorityClass, cfg.unityCatalogImagePath(), containers) if err != nil { - return 0, "", false, err + return 0, "", err } // A generated token is never a repeat. Container runs carry their command // paths on containers, which the SDK's DeploymentSpec does not model. - if userIdempotencyToken(idempotencyKey, cfg) != "" && len(containers) == 0 { - existing = isExistingRun(ctx, w, runID, commandPath) + if existing, ok := ctx.Value(existingRunKey{}).(*bool); ok && userIdempotencyToken(idempotencyKey, cfg) != "" && len(containers) == 0 { + *existing = isExistingRun(ctx, w, runID, commandPath) } dashboardURL = strings.TrimRight(w.Config.Host, "/") + "/jobs/runs/" + strconv.FormatInt(runID, 10) - return runID, dashboardURL, existing, nil + return runID, dashboardURL, nil } func submittedContainers(cfg *runConfig, funcDir string) []submittedContainer { diff --git a/cmd/air/runsubmit_test.go b/cmd/air/runsubmit_test.go index ff5128c2f04..441a1c12d26 100644 --- a/cmd/air/runsubmit_test.go +++ b/cmd/air/runsubmit_test.go @@ -379,7 +379,7 @@ func TestSubmitWorkload(t *testing.T) { cfg, err := loadRunConfig(cfgPath) require.NoError(t, err) - runID, dashboardURL, _, err := submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) + runID, dashboardURL, err := submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) require.NoError(t, err) assert.Equal(t, int64(777), runID) assert.Contains(t, dashboardURL, "/jobs/runs/777") @@ -448,7 +448,7 @@ func TestSubmitWorkloadValidationOutcome(t *testing.T) { require.NoError(t, err) ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - _, _, _, err = submitWorkload(ctx, w, cfg, cfgPath, "idem-key", false) + _, _, err = submitWorkload(ctx, w, cfg, cfgPath, "idem-key", false) if tt.wantError != "" { require.ErrorContains(t, err, tt.wantError) } else { @@ -499,7 +499,7 @@ func TestSubmitWorkloadValidatesWhileStaging(t *testing.T) { cfg, err := loadRunConfig(cfgPath) require.NoError(t, err) - runID, _, _, err := submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) + runID, _, err := submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) require.NoError(t, err) assert.Equal(t, int64(777), runID) } @@ -535,7 +535,7 @@ func TestSubmitWorkloadValidationFailureCancelsStaging(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() - _, _, _, err = submitWorkload(ctx, w, cfg, cfgPath, "idem-key", false) + _, _, err = submitWorkload(ctx, w, cfg, cfgPath, "idem-key", false) require.ErrorContains(t, err, "task.experiment: is invalid") require.NoError(t, ctx.Err(), "staging was not cancelled by the validation failure") assert.Zero(t, submissions.Load()) @@ -562,7 +562,7 @@ func TestSubmitWorkloadStagingErrorPreventsSubmit(t *testing.T) { cfg, err := loadRunConfig(cfgPath) require.NoError(t, err) - _, _, _, err = submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) + _, _, err = submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) require.Error(t, err) assert.ErrorContains(t, err, "failed to create launch directory") assert.Zero(t, submitCalls.Load()) @@ -592,7 +592,7 @@ func TestSubmitWorkloadHonorsOverride(t *testing.T) { cfg, err := loadRunConfigWithOverrides(t.Context(), cfgPath, []string{"compute.num_accelerators=4"}) require.NoError(t, err) - _, _, _, err = submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) + _, _, err = submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) require.NoError(t, err) require.Len(t, got.Tasks, 1) @@ -623,7 +623,7 @@ environment: cfg, err := loadRunConfig(cfgPath) require.NoError(t, err) - _, _, _, err = submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) + _, _, err = submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) require.NoError(t, err) tasks, ok := got["tasks"].([]any) @@ -696,7 +696,7 @@ containers: `) cfg, err := loadRunConfig(cfgPath) require.NoError(t, err) - _, _, _, err = submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) + _, _, err = submitWorkload(t.Context(), w, cfg, cfgPath, "idem-key", false) require.NoError(t, err) aiRuntimeTask, err := aiRuntimeTaskFromSubmitBody(got) @@ -752,7 +752,7 @@ code_source: // The upload path logs via cmdio; the real `air run` context carries it. ctx := cmdio.MockDiscard(t.Context()) - _, _, _, err = submitWorkload(ctx, w, loaded, cfgPath, "idem", false) + _, _, err = submitWorkload(ctx, w, loaded, cfgPath, "idem", false) require.NoError(t, err) at := got.Tasks[0].AiRuntimeTask @@ -797,7 +797,7 @@ code_source: require.NoError(t, err) ctx := cmdio.MockDiscard(t.Context()) - _, _, _, err = submitWorkload(ctx, w, loaded, cfgPath, "idem", false) + _, _, err = submitWorkload(ctx, w, loaded, cfgPath, "idem", false) require.NoError(t, err) at := got.Tasks[0].AiRuntimeTask @@ -1040,7 +1040,7 @@ code_source: require.NoError(t, err) ctx := cmdio.MockDiscard(t.Context()) - _, _, _, err = submitWorkload(ctx, w, loaded, cfgPath, "idem", false) + _, _, err = submitWorkload(ctx, w, loaded, cfgPath, "idem", false) require.NoError(t, err) at := got.Tasks[0].AiRuntimeTask @@ -1072,7 +1072,7 @@ func TestSubmitWorkloadGuards(t *testing.T) { cfg := *base cfg.UsagePolicyName = new("nope") - _, _, _, err = submitWorkload(t.Context(), pw, &cfg, cfgPath, "", false) + _, _, err = submitWorkload(t.Context(), pw, &cfg, cfgPath, "", false) require.ErrorContains(t, err, `no usage policy named "nope"`) assert.Zero(t, submissions.Load()) }) @@ -1094,7 +1094,7 @@ func TestSubmitWorkloadGuards(t *testing.T) { cfg := *base cfg.UsagePolicyName = new("nope") - _, _, _, err = submitWorkload(t.Context(), pw, &cfg, cfgPath, "", false) + _, _, err = submitWorkload(t.Context(), pw, &cfg, cfgPath, "", false) require.ErrorContains(t, err, "task.experiment: is invalid") assert.Zero(t, policyLookups.Load()) }) @@ -1129,7 +1129,7 @@ func TestSubmitWorkloadSendsUsagePolicy(t *testing.T) { cfg, err := loadRunConfig(cfgPath) require.NoError(t, err) - _, _, _, err = submitWorkload(cmdio.MockDiscard(t.Context()), w, cfg, cfgPath, "idem", false) + _, _, err = submitWorkload(cmdio.MockDiscard(t.Context()), w, cfg, cfgPath, "idem", false) require.NoError(t, err) assert.Equal(t, policyID, got.BudgetPolicyId) }) @@ -1140,7 +1140,7 @@ func TestSubmitWorkloadSendsUsagePolicy(t *testing.T) { cfg, err := loadRunConfig(cfgPath) require.NoError(t, err) - _, _, _, err = submitWorkload(cmdio.MockDiscard(t.Context()), w, cfg, cfgPath, "idem", false) + _, _, err = submitWorkload(cmdio.MockDiscard(t.Context()), w, cfg, cfgPath, "idem", false) require.NoError(t, err) assert.Equal(t, policyID, got.BudgetPolicyId) }) diff --git a/cmd/air/telemetry_test.go b/cmd/air/telemetry_test.go index d57e59c09b7..35a0cad984d 100644 --- a/cmd/air/telemetry_test.go +++ b/cmd/air/telemetry_test.go @@ -89,7 +89,7 @@ func TestAirRunTelemetry(t *testing.T) { cfgPath := writeConfigFile(t, "run.yaml", configYAML) cfg, err := loadRunConfig(cfgPath) require.NoError(t, err) - _, _, _, err = submitWorkload(ctx, w, cfg, cfgPath, "idem", false) + _, _, err = submitWorkload(ctx, w, cfg, cfgPath, "idem", false) if tc.fail || tc.failUpload { require.Error(t, err) } else { From e9d3f9358c5dfc1879ad0b8c67aaa197e5eaf9c3 Mon Sep 17 00:00:00 2001 From: Abdul-Kareem Aliu Date: Thu, 12 Nov 2026 00:06:28 +0000 Subject: [PATCH 4/6] Point changelog entry at #7529 Co-authored-by: Isaac --- .nextchanges/air/idempotency-existing-run.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/air/idempotency-existing-run.md b/.nextchanges/air/idempotency-existing-run.md index 5f854e8d002..6d732bdf118 100644 --- a/.nextchanges/air/idempotency-existing-run.md +++ b/.nextchanges/air/idempotency-existing-run.md @@ -1 +1 @@ -* `air run` now says when `--idempotency-key` or `idempotency_token` returned an existing run, and its JSON output reports this as `existing`. ([#7527](https://github.com/databricks/cli/pull/7527)) +* `air run` now says when `--idempotency-key` or `idempotency_token` returned an existing run, and its JSON output reports this as `existing`. ([#7529](https://github.com/databricks/cli/pull/7529)) From ea9d714874603beb441d486bfe2daa10b525f122 Mon Sep 17 00:00:00 2001 From: Abdul-Kareem Aliu Date: Thu, 12 Nov 2026 00:24:46 +0000 Subject: [PATCH 5/6] air: Cover existing-run reporting and skip the lookup under --watch -o json The --watch JSON events never carry `existing`, so the runs/get that detects a reused idempotency key was wasted there. Keep it for text output (with or without --watch) and the non-watch JSON result. Add tests for --watch text output printing the existing-run line before the streamed logs, the per-mode runs/get lookup count, the runs/get failure and missing command path fallbacks, and the containers skip. Co-authored-by: Isaac --- cmd/air/run.go | 6 ++- cmd/air/run_test.go | 127 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 1 deletion(-) diff --git a/cmd/air/run.go b/cmd/air/run.go index 9009a2a02e9..0a328f568bd 100644 --- a/cmd/air/run.go +++ b/cmd/air/run.go @@ -189,7 +189,11 @@ The path must be a separate argument: cobra reserves -h as a boolean, so } w := cmdctx.WorkspaceClient(ctx) - submitCtx, existing := withExistingRunReport(ctx) + // The --watch JSON events don't report existing, so skip its runs/get there. + submitCtx, existing := ctx, new(bool) + if !watch || !jsonOut { + submitCtx, existing = withExistingRunReport(ctx) + } runID, dashboardURL, err := submitWorkload(submitCtx, w, cfg, file, idempotencyKey, !jsonOut) if err != nil { return err diff --git a/cmd/air/run_test.go b/cmd/air/run_test.go index f26924f6319..7b8ca809259 100644 --- a/cmd/air/run_test.go +++ b/cmd/air/run_test.go @@ -210,6 +210,133 @@ func TestRunSubmitJSONReportsExistingRun(t *testing.T) { assert.Equal(t, "PENDING", second.Status) } +// earlierLaunchRunGet is a terminated run whose command path is from an earlier +// submit's launch directory, so a submit with a user key sees it as existing. +const earlierLaunchRunGet = `{ + "run_id": 777, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "tasks": [{ + "run_id": 778, + "attempt_number": 0, + "ai_runtime_task": {"deployments": [{"command_path": "/Users/u@example.com/.air/cli_launch/my-run/my-run_0123456789abcdef/command.sh"}]} + }] +}` + +// existingRunServer serves `air run` with or without --watch. runs/get replies +// with runGetStatus and runGetBody; the returned counter tracks its calls. +func existingRunServer(t *testing.T, runGetStatus int, runGetBody string) (string, *atomic.Int32) { + t.Helper() + var runGets atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/jobs/runs/submit"): + _, _ = w.Write([]byte(`{"run_id": 777}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get": + runGets.Add(1) + w.WriteHeader(runGetStatus) + _, _ = w.Write([]byte(runGetBody)) + case strings.HasPrefix(r.URL.Path, "/api/2.0/ai-training/workflows/by-run-id/"): + _, _ = w.Write([]byte(`{"log_records": [ + {"time_unix_nano": 1700000002000000000, "body": "step 2", "node_index": 0}, + {"time_unix_nano": 1700000001000000000, "body": "step 1", "node_index": 0} + ]}`)) + default: + _, _ = w.Write([]byte(`{"userName": "u@example.com", "workspace_id": 1}`)) + } + })) + t.Cleanup(srv.Close) + return srv.URL, &runGets +} + +func TestRunWatchExistingRunPrintsBeforeLogs(t *testing.T) { + srvURL, _ := existingRunServer(t, http.StatusOK, earlierLaunchRunGet) + + out := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--watch", "--idempotency-key", "watch-key") + existingLine := `Returned existing workload with Job Run ID: 777 (idempotency key "watch-key" was already used)` + require.Contains(t, out, existingLine) + require.Contains(t, out, "step 1\nstep 2") + assert.Less(t, strings.Index(out, existingLine), strings.Index(out, "step 1")) + assert.NotContains(t, out, "Submitted workload") +} + +func TestRunExistingRunLookupByOutputMode(t *testing.T) { + for _, tc := range []struct { + name string + out flags.Output + args []string + // wantLookups is the runs/get calls a user key adds over a generated token. + wantLookups int32 + }{ + {name: "text", out: flags.OutputText, wantLookups: 1}, + {name: "json", out: flags.OutputJSON, wantLookups: 1}, + {name: "watch text", out: flags.OutputText, args: []string{"--watch"}, wantLookups: 1}, + // The --watch JSON events don't report existing, so there is no lookup. + {name: "watch json", out: flags.OutputJSON, args: []string{"--watch"}, wantLookups: 0}, + } { + t.Run(tc.name, func(t *testing.T) { + generatedURL, generatedRunGets := existingRunServer(t, http.StatusOK, earlierLaunchRunGet) + runIdempotentSubmit(t, generatedURL, tc.out, minimalConfig, tc.args...) + + userKeyURL, userKeyRunGets := existingRunServer(t, http.StatusOK, earlierLaunchRunGet) + runIdempotentSubmit(t, userKeyURL, tc.out, minimalConfig, append(tc.args, "--idempotency-key", "mode-key")...) + + assert.Equal(t, tc.wantLookups, userKeyRunGets.Load()-generatedRunGets.Load()) + }) + } +} + +func TestRunSubmitExistingRunLookupTreatsUnknownAsNew(t *testing.T) { + for _, tc := range []struct { + name string + runGetStatus int + runGetBody string + }{ + { + name: "runs/get fails", + runGetStatus: http.StatusNotFound, + runGetBody: `{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "run not found"}`, + }, + { + name: "no deployment", + runGetStatus: http.StatusOK, + runGetBody: `{"run_id": 777, "tasks": [{"run_id": 778, "ai_runtime_task": {}}]}`, + }, + { + name: "empty command path", + runGetStatus: http.StatusOK, + runGetBody: `{"run_id": 777, "tasks": [{"run_id": 778, "ai_runtime_task": {"deployments": [{}]}}]}`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + srvURL, runGets := existingRunServer(t, tc.runGetStatus, tc.runGetBody) + + out := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--idempotency-key", "unknown-key") + assert.Contains(t, out, "Submitted workload with Job Run ID: 777") + assert.NotContains(t, out, "Returned existing workload") + assert.Equal(t, int32(1), runGets.Load()) + }) + } +} + +func TestRunSubmitContainersSkipsExistingRunLookup(t *testing.T) { + srvURL, runGets := existingRunServer(t, http.StatusOK, earlierLaunchRunGet) + config := ` +experiment_name: my-run +compute: + accelerator_type: GPU_1xH100 + num_accelerators: 1 +containers: + - name: inference + command: python infer.py + ranks: [0] + unity_catalog_image: main.ml.inference:v1 +` + + out := runIdempotentSubmit(t, srvURL, flags.OutputText, config, "--idempotency-key", "container-key") + assert.Contains(t, out, "Submitted workload with Job Run ID: 777") + assert.Zero(t, runGets.Load()) +} + type dryRunRequest struct { method string path string From 6e31540d1b2784802ebe9272b58a8cef28b3d4dd Mon Sep 17 00:00:00 2001 From: Abdul-Kareem Aliu Date: Thu, 12 Nov 2026 00:47:03 +0000 Subject: [PATCH 6/6] air: Report unknown existing-run state and compare launch dirs The existing-run report is now tri-state: JSON omits `existing` when the CLI could not tell (runs/get failed or timed out, no deployment, an unrecognized command path, or a container run) instead of claiming false. Generated-token submits still report false, since they can never repeat. The lookup compares only the unique _ launch directory, so a different workspace path prefix in the runs/get echo cannot turn a fresh run into an existing one, and is bounded by a 5s timeout. Co-authored-by: Isaac --- .nextchanges/air/idempotency-existing-run.md | 2 +- cmd/air/run.go | 11 +- cmd/air/run_test.go | 120 ++++++++++++++----- cmd/air/runlaunch.go | 17 +++ cmd/air/runsubmit.go | 63 +++++++--- libs/testserver/jobs.go | 1 + 6 files changed, 156 insertions(+), 58 deletions(-) diff --git a/.nextchanges/air/idempotency-existing-run.md b/.nextchanges/air/idempotency-existing-run.md index 6d732bdf118..fb81df3ad13 100644 --- a/.nextchanges/air/idempotency-existing-run.md +++ b/.nextchanges/air/idempotency-existing-run.md @@ -1 +1 @@ -* `air run` now says when `--idempotency-key` or `idempotency_token` returned an existing run, and its JSON output reports this as `existing`. ([#7529](https://github.com/databricks/cli/pull/7529)) +* `air run` now says when `--idempotency-key` or `idempotency_token` returned an existing run, and its `-o json` result reports this as `existing` (the `--watch` JSONL events do not). ([#7529](https://github.com/databricks/cli/pull/7529)) diff --git a/cmd/air/run.go b/cmd/air/run.go index 0a328f568bd..1c298051c0c 100644 --- a/cmd/air/run.go +++ b/cmd/air/run.go @@ -33,7 +33,8 @@ type runResult struct { DryRun bool `json:"dry_run,omitempty"` RunID string `json:"run_id,omitempty"` DashboardURL string `json:"dashboard_url,omitempty"` - // Existing is nil for a dry run, which submits nothing. + // Existing is nil when unknown: a dry run submits nothing, and the + // existing-run lookup can fail or not apply (e.g. container runs). Existing *bool `json:"existing,omitempty"` } @@ -190,16 +191,16 @@ The path must be a separate argument: cobra reserves -h as a boolean, so w := cmdctx.WorkspaceClient(ctx) // The --watch JSON events don't report existing, so skip its runs/get there. - submitCtx, existing := ctx, new(bool) + submitCtx, report := ctx, &existingRunReport{} if !watch || !jsonOut { - submitCtx, existing = withExistingRunReport(ctx) + submitCtx, report = withExistingRunReport(ctx) } runID, dashboardURL, err := submitWorkload(submitCtx, w, cfg, file, idempotencyKey, !jsonOut) if err != nil { return err } existingKey := "" - if *existing { + if report.Existing != nil && *report.Existing { existingKey = userIdempotencyToken(idempotencyKey, cfg) } @@ -215,7 +216,7 @@ The path must be a separate argument: cobra reserves -h as a boolean, so } // PENDING is the submit status, distinct from the --watch JSONL // SUBMITTED event type below. - if err := renderEnvelope(ctx, runResult{Status: "PENDING", RunID: runIDStr, DashboardURL: dashboardURL, Existing: existing}); err != nil { + if err := renderEnvelope(ctx, runResult{Status: "PENDING", RunID: runIDStr, DashboardURL: dashboardURL, Existing: report.Existing}); err != nil { return err } grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, false) diff --git a/cmd/air/run_test.go b/cmd/air/run_test.go index 7b8ca809259..0fe50b3336e 100644 --- a/cmd/air/run_test.go +++ b/cmd/air/run_test.go @@ -3,6 +3,7 @@ package aircmd import ( "bytes" "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" @@ -15,6 +16,7 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -192,19 +194,13 @@ func TestRunSubmitYAMLIdempotencyTokenReturnsExistingRun(t *testing.T) { func TestRunSubmitJSONReportsExistingRun(t *testing.T) { srvURL := idempotentSubmitServer(t) - decode := func(out string) runResult { - var envelope struct { - Data runResult `json:"data"` - } - require.NoError(t, json.Unmarshal([]byte(out), &envelope), out) - require.NotNil(t, envelope.Data.Existing) - return envelope.Data - } - first := decode(runIdempotentSubmit(t, srvURL, flags.OutputJSON, minimalConfig, "--idempotency-key", "json-key")) + first := decodeRunResult(t, runIdempotentSubmit(t, srvURL, flags.OutputJSON, minimalConfig, "--idempotency-key", "json-key")) + require.NotNil(t, first.Existing) assert.False(t, *first.Existing) - second := decode(runIdempotentSubmit(t, srvURL, flags.OutputJSON, minimalConfig, "--idempotency-key", "json-key")) + second := decodeRunResult(t, runIdempotentSubmit(t, srvURL, flags.OutputJSON, minimalConfig, "--idempotency-key", "json-key")) + require.NotNil(t, second.Existing) assert.True(t, *second.Existing) assert.Equal(t, first.RunID, second.RunID) assert.Equal(t, "PENDING", second.Status) @@ -222,19 +218,35 @@ const earlierLaunchRunGet = `{ }] }` +// runGetReply answers runs/get; submittedCommandPath is the command path the +// preceding runs/submit sent. +type runGetReply func(w http.ResponseWriter, r *http.Request, submittedCommandPath string) + +func staticRunGet(status int, body string) runGetReply { + return func(w http.ResponseWriter, _ *http.Request, _ string) { + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + } +} + // existingRunServer serves `air run` with or without --watch. runs/get replies -// with runGetStatus and runGetBody; the returned counter tracks its calls. -func existingRunServer(t *testing.T, runGetStatus int, runGetBody string) (string, *atomic.Int32) { +// via runGet; the returned counter tracks its calls. +func existingRunServer(t *testing.T, runGet runGetReply) (string, *atomic.Int32) { t.Helper() var runGets atomic.Int32 + var submittedCommandPath atomic.Value + submittedCommandPath.Store("") srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case strings.HasSuffix(r.URL.Path, "/jobs/runs/submit"): + var req jobs.SubmitRun + if json.NewDecoder(r.Body).Decode(&req) == nil && len(req.Tasks) > 0 && req.Tasks[0].AiRuntimeTask != nil && len(req.Tasks[0].AiRuntimeTask.Deployments) > 0 { + submittedCommandPath.Store(req.Tasks[0].AiRuntimeTask.Deployments[0].CommandPath) + } _, _ = w.Write([]byte(`{"run_id": 777}`)) case r.URL.Path == "/api/2.2/jobs/runs/get": runGets.Add(1) - w.WriteHeader(runGetStatus) - _, _ = w.Write([]byte(runGetBody)) + runGet(w, r, submittedCommandPath.Load().(string)) case strings.HasPrefix(r.URL.Path, "/api/2.0/ai-training/workflows/by-run-id/"): _, _ = w.Write([]byte(`{"log_records": [ {"time_unix_nano": 1700000002000000000, "body": "step 2", "node_index": 0}, @@ -248,8 +260,18 @@ func existingRunServer(t *testing.T, runGetStatus int, runGetBody string) (strin return srv.URL, &runGets } +// decodeRunResult returns the data of a non-watch `air run -o json` envelope. +func decodeRunResult(t *testing.T, out string) runResult { + t.Helper() + var envelope struct { + Data runResult `json:"data"` + } + require.NoError(t, json.Unmarshal([]byte(out), &envelope), out) + return envelope.Data +} + func TestRunWatchExistingRunPrintsBeforeLogs(t *testing.T) { - srvURL, _ := existingRunServer(t, http.StatusOK, earlierLaunchRunGet) + srvURL, _ := existingRunServer(t, staticRunGet(http.StatusOK, earlierLaunchRunGet)) out := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--watch", "--idempotency-key", "watch-key") existingLine := `Returned existing workload with Job Run ID: 777 (idempotency key "watch-key" was already used)` @@ -274,10 +296,10 @@ func TestRunExistingRunLookupByOutputMode(t *testing.T) { {name: "watch json", out: flags.OutputJSON, args: []string{"--watch"}, wantLookups: 0}, } { t.Run(tc.name, func(t *testing.T) { - generatedURL, generatedRunGets := existingRunServer(t, http.StatusOK, earlierLaunchRunGet) + generatedURL, generatedRunGets := existingRunServer(t, staticRunGet(http.StatusOK, earlierLaunchRunGet)) runIdempotentSubmit(t, generatedURL, tc.out, minimalConfig, tc.args...) - userKeyURL, userKeyRunGets := existingRunServer(t, http.StatusOK, earlierLaunchRunGet) + userKeyURL, userKeyRunGets := existingRunServer(t, staticRunGet(http.StatusOK, earlierLaunchRunGet)) runIdempotentSubmit(t, userKeyURL, tc.out, minimalConfig, append(tc.args, "--idempotency-key", "mode-key")...) assert.Equal(t, tc.wantLookups, userKeyRunGets.Load()-generatedRunGets.Load()) @@ -285,41 +307,70 @@ func TestRunExistingRunLookupByOutputMode(t *testing.T) { } } -func TestRunSubmitExistingRunLookupTreatsUnknownAsNew(t *testing.T) { +func TestRunSubmitExistingRunMatchesLaunchDirAcrossPathPrefixes(t *testing.T) { + // The backend echoes this submit's own command path in another prefix form. + srvURL, _ := existingRunServer(t, func(w http.ResponseWriter, _ *http.Request, submitted string) { + echoed := strings.TrimPrefix(submitted, "/Workspace") + require.NotEqual(t, submitted, echoed) + _, _ = fmt.Fprintf(w, `{"run_id": 777, "tasks": [{"run_id": 778, "ai_runtime_task": {"deployments": [{"command_path": %q}]}}]}`, echoed) + }) + + out := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--idempotency-key", "prefix-key") + assert.Contains(t, out, "Submitted workload with Job Run ID: 777") + + result := decodeRunResult(t, runIdempotentSubmit(t, srvURL, flags.OutputJSON, minimalConfig, "--idempotency-key", "prefix-key")) + require.NotNil(t, result.Existing) + assert.False(t, *result.Existing) +} + +func TestRunSubmitExistingRunLookupReportsUnknown(t *testing.T) { for _, tc := range []struct { - name string - runGetStatus int - runGetBody string + name string + runGet runGetReply }{ { - name: "runs/get fails", - runGetStatus: http.StatusNotFound, - runGetBody: `{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "run not found"}`, + name: "runs/get fails", + runGet: staticRunGet(http.StatusNotFound, `{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "run not found"}`), }, { - name: "no deployment", - runGetStatus: http.StatusOK, - runGetBody: `{"run_id": 777, "tasks": [{"run_id": 778, "ai_runtime_task": {}}]}`, + name: "runs/get times out", + runGet: func(_ http.ResponseWriter, r *http.Request, _ string) { + <-r.Context().Done() + }, }, { - name: "empty command path", - runGetStatus: http.StatusOK, - runGetBody: `{"run_id": 777, "tasks": [{"run_id": 778, "ai_runtime_task": {"deployments": [{}]}}]}`, + name: "no deployment", + runGet: staticRunGet(http.StatusOK, `{"run_id": 777, "tasks": [{"run_id": 778, "ai_runtime_task": {}}]}`), + }, + { + name: "empty command path", + runGet: staticRunGet(http.StatusOK, `{"run_id": 777, "tasks": [{"run_id": 778, "ai_runtime_task": {"deployments": [{}]}}]}`), + }, + { + name: "unrecognized command path", + runGet: staticRunGet(http.StatusOK, `{"run_id": 777, "tasks": [{"run_id": 778, "ai_runtime_task": {"deployments": [{"command_path": "/Users/u@example.com/train.sh"}]}}]}`), }, } { t.Run(tc.name, func(t *testing.T) { - srvURL, runGets := existingRunServer(t, tc.runGetStatus, tc.runGetBody) + defaultTimeout := existingRunLookupTimeout + existingRunLookupTimeout = 50 * time.Millisecond + t.Cleanup(func() { existingRunLookupTimeout = defaultTimeout }) + srvURL, runGets := existingRunServer(t, tc.runGet) out := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--idempotency-key", "unknown-key") assert.Contains(t, out, "Submitted workload with Job Run ID: 777") assert.NotContains(t, out, "Returned existing workload") - assert.Equal(t, int32(1), runGets.Load()) + + jsonOut := runIdempotentSubmit(t, srvURL, flags.OutputJSON, minimalConfig, "--idempotency-key", "unknown-key") + assert.Nil(t, decodeRunResult(t, jsonOut).Existing) + assert.NotContains(t, jsonOut, `"existing"`) + assert.Equal(t, int32(2), runGets.Load()) }) } } func TestRunSubmitContainersSkipsExistingRunLookup(t *testing.T) { - srvURL, runGets := existingRunServer(t, http.StatusOK, earlierLaunchRunGet) + srvURL, runGets := existingRunServer(t, staticRunGet(http.StatusOK, earlierLaunchRunGet)) config := ` experiment_name: my-run compute: @@ -334,6 +385,9 @@ containers: out := runIdempotentSubmit(t, srvURL, flags.OutputText, config, "--idempotency-key", "container-key") assert.Contains(t, out, "Submitted workload with Job Run ID: 777") + + jsonOut := runIdempotentSubmit(t, srvURL, flags.OutputJSON, config, "--idempotency-key", "container-key") + assert.NotContains(t, jsonOut, `"existing"`) assert.Zero(t, runGets.Load()) } diff --git a/cmd/air/runlaunch.go b/cmd/air/runlaunch.go index a888d80d85d..c67345d8534 100644 --- a/cmd/air/runlaunch.go +++ b/cmd/air/runlaunch.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path" + "regexp" "strings" "uuid" @@ -58,6 +59,22 @@ func cliLaunchDir(base, experiment, run string) string { return path.Join(base, ".air", "cli_launch", experiment, run+"_"+unique) } +// launchDirPattern matches the _ directory name cliLaunchDir creates. +var launchDirPattern = regexp.MustCompile(`_[0-9a-f]{16}$`) + +// launchDirName returns the unique launch directory that holds commandPath's +// command script. Only this segment identifies a launch: the prefix can differ +// in form (/Users/... vs /Workspace/Users/...) between the submitted path and +// the one runs/get echoes. +func launchDirName(commandPath string) (string, bool) { + dir, file := path.Split(commandPath) + name := path.Base(dir) + if file != commandScriptName || !launchDirPattern.MatchString(name) { + return "", false + } + return name, true +} + // prospectiveLaunchPaths returns the workspace paths a submission would use // without creating directories or uploading artifacts. func prospectiveLaunchPaths(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig) (string, string, string, error) { diff --git a/cmd/air/runsubmit.go b/cmd/air/runsubmit.go index 04bd5720ad6..63df0f7dfd0 100644 --- a/cmd/air/runsubmit.go +++ b/cmd/air/runsubmit.go @@ -289,30 +289,49 @@ func effectiveIdempotencyToken(flag string, cfg *runConfig) string { type existingRunKey struct{} -// withExistingRunReport returns a context in which submitWorkload sets the -// returned bool when a caller-supplied idempotency token returned an existing -// run. Only the run command opts in, as the check costs an extra runs/get. -func withExistingRunReport(ctx context.Context) (context.Context, *bool) { - existing := new(bool) - return context.WithValue(ctx, existingRunKey{}, existing), existing +// existingRunReport records whether a submit returned an existing run. +// Existing is nil when the CLI could not tell. +type existingRunReport struct { + Existing *bool } +// withExistingRunReport returns a context in which submitWorkload fills the +// returned report. A context value avoids changing submitWorkload's signature +// for its many test callers, and lets callers opt in to the extra runs/get only +// where the result is shown. +func withExistingRunReport(ctx context.Context) (context.Context, *existingRunReport) { + report := &existingRunReport{} + return context.WithValue(ctx, existingRunKey{}, report), report +} + +// existingRunLookupTimeout bounds the runs/get behind the existing-run report: +// the report only refines the success line, so a slow backend must not hold +// it up. A var so tests can shorten it. +var existingRunLookupTimeout = 5 * time.Second + // isExistingRun reports whether runID was started by an earlier submit that -// used the same idempotency token. runs/submit returns only a run id, so the -// run's command path is the signal: every submit uploads to a fresh launch -// directory, and a fresh run references this submit's commandPath while an -// existing run references an earlier one's. -func isExistingRun(ctx context.Context, w *databricks.WorkspaceClient, runID int64, commandPath string) bool { +// used the same idempotency token, or nil when that can't be determined. +// runs/submit returns only a run id, so the run's command path is the signal: +// every submit uploads to a fresh launch directory, and a fresh run references +// this submit's commandPath while an existing run references an earlier one's. +func isExistingRun(ctx context.Context, w *databricks.WorkspaceClient, runID int64, commandPath string) *bool { + ctx, cancel := context.WithTimeout(ctx, existingRunLookupTimeout) + defer cancel() run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: runID}) if err != nil { log.Warnf(ctx, "could not check whether run %d already existed: %v", runID, err) - return false + return nil } task := aiRuntimeTaskOf(run) - if task == nil || len(task.Deployments) == 0 || task.Deployments[0].CommandPath == "" { - return false + if task == nil || len(task.Deployments) == 0 { + return nil } - return task.Deployments[0].CommandPath != commandPath + got, gotOK := launchDirName(task.Deployments[0].CommandPath) + want, wantOK := launchDirName(commandPath) + if !gotOK || !wantOK { + return nil + } + return new(got != want) } // validateIdempotencyToken keeps the CLI's submission guard as a fallback when @@ -481,10 +500,16 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run return 0, "", err } - // A generated token is never a repeat. Container runs carry their command - // paths on containers, which the SDK's DeploymentSpec does not model. - if existing, ok := ctx.Value(existingRunKey{}).(*bool); ok && userIdempotencyToken(idempotencyKey, cfg) != "" && len(containers) == 0 { - *existing = isExistingRun(ctx, w, runID, commandPath) + // A generated token is never a repeat, so that run is definitely new. + // Container runs stay unknown: they carry their command paths on containers, + // which the SDK's DeploymentSpec does not model. + if report, ok := ctx.Value(existingRunKey{}).(*existingRunReport); ok { + switch { + case userIdempotencyToken(idempotencyKey, cfg) == "": + report.Existing = new(false) + case len(containers) == 0: + report.Existing = isExistingRun(ctx, w, runID, commandPath) + } } dashboardURL = strings.TrimRight(w.Config.Host, "/") + "/jobs/runs/" + strconv.FormatInt(runID, 10) diff --git a/libs/testserver/jobs.go b/libs/testserver/jobs.go index 8c4669c945f..20905dac80e 100644 --- a/libs/testserver/jobs.go +++ b/libs/testserver/jobs.go @@ -648,6 +648,7 @@ func (s *FakeWorkspace) JobsSubmit(req Request) Response { defer s.LockUnlock()() // A resent request with the same non-empty token returns the run that token started. + // JobRunIdempotency is shared by runs/run-now and runs/submit. if request.IdempotencyToken != "" { if runId, ok := s.JobRunIdempotency[request.IdempotencyToken]; ok { return Response{Body: jobs.SubmitRunResponse{RunId: runId}}