diff --git a/.nextchanges/air/idempotency-existing-run.md b/.nextchanges/air/idempotency-existing-run.md new file mode 100644 index 00000000000..5f854e8d002 --- /dev/null +++ b/.nextchanges/air/idempotency-existing-run.md @@ -0,0 +1 @@ +* `air run` now says when `--idempotency-key` or `idempotency_token` returned an existing run, and its JSON output reports this as `existing`. ([#7527](https://github.com/databricks/cli/pull/7527)) diff --git a/acceptance/air/run-idempotency/databricks.yml b/acceptance/air/run-idempotency/databricks.yml new file mode 100644 index 00000000000..a1c79b6244a --- /dev/null +++ b/acceptance/air/run-idempotency/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: air-run-idempotency diff --git a/acceptance/air/run-idempotency/out.test.toml b/acceptance/air/run-idempotency/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/air/run-idempotency/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/air/run-idempotency/output.txt b/acceptance/air/run-idempotency/output.txt new file mode 100644 index 00000000000..d53421f4022 --- /dev/null +++ b/acceptance/air/run-idempotency/output.txt @@ -0,0 +1,52 @@ + +=== first submit with a key starts a run + +>>> [CLI] air run -f run.yaml --idempotency-key flag-key +Submitting experiment: idempotency-smoke +Submitted workload with Job Run ID: [FLAG_KEY_RUN_ID] +View job run at: [DATABRICKS_URL]/jobs/runs/[FLAG_KEY_RUN_ID] + +Tip: use --watch when submitting a run to stream logs to your terminal. +Stream logs after submission using: + databricks air logs [FLAG_KEY_RUN_ID] + +=== resubmitting with the same key returns the existing run + +>>> [CLI] air run -f run.yaml --idempotency-key flag-key +Submitting experiment: idempotency-smoke +Returned existing workload with Job Run ID: [FLAG_KEY_RUN_ID] (idempotency key "flag-key" was already used) +View job run at: [DATABRICKS_URL]/jobs/runs/[FLAG_KEY_RUN_ID] + +Tip: use --watch when submitting a run to stream logs to your terminal. +Stream logs after submission using: + databricks air logs [FLAG_KEY_RUN_ID] + +=== JSON output reports the existing run + +>>> [CLI] air run -f run.yaml --idempotency-key flag-key -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "status": "PENDING", + "run_id": "[FLAG_KEY_RUN_ID]", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/[FLAG_KEY_RUN_ID]", + "existing": true + } +} + +=== a different key starts a new run + +>>> [CLI] air run -f run.yaml --idempotency-key other-key -o json +false + +=== a YAML idempotency_token is deduplicated the same way + +>>> [CLI] air run -f run-token.yaml +Submitting experiment: idempotency-smoke +Returned existing workload with Job Run ID: [YAML_TOKEN_RUN_ID] (idempotency key "yaml-key" was already used) +View job run at: [DATABRICKS_URL]/jobs/runs/[YAML_TOKEN_RUN_ID] + +Tip: use --watch when submitting a run to stream logs to your terminal. +Stream logs after submission using: + databricks air logs [YAML_TOKEN_RUN_ID] diff --git a/acceptance/air/run-idempotency/run-token.yaml b/acceptance/air/run-idempotency/run-token.yaml new file mode 100644 index 00000000000..63dd8d7b961 --- /dev/null +++ b/acceptance/air/run-idempotency/run-token.yaml @@ -0,0 +1,6 @@ +experiment_name: idempotency-smoke +command: python train.py +compute: + accelerator_type: GPU_1xH100 + num_accelerators: 1 +idempotency_token: yaml-key diff --git a/acceptance/air/run-idempotency/run.yaml b/acceptance/air/run-idempotency/run.yaml new file mode 100644 index 00000000000..38eecf38446 --- /dev/null +++ b/acceptance/air/run-idempotency/run.yaml @@ -0,0 +1,5 @@ +experiment_name: idempotency-smoke +command: python train.py +compute: + accelerator_type: GPU_1xH100 + num_accelerators: 1 diff --git a/acceptance/air/run-idempotency/script b/acceptance/air/run-idempotency/script new file mode 100644 index 00000000000..146e3d8521b --- /dev/null +++ b/acceptance/air/run-idempotency/script @@ -0,0 +1,16 @@ +title "first submit with a key starts a run" +trace $CLI air run -f run.yaml --idempotency-key flag-key +add_repl.py "$($CLI air run -f run.yaml --idempotency-key flag-key -o json | jq -r .data.run_id)" FLAG_KEY_RUN_ID + +title "resubmitting with the same key returns the existing run" +trace $CLI air run -f run.yaml --idempotency-key flag-key + +title "JSON output reports the existing run" +trace $CLI air run -f run.yaml --idempotency-key flag-key -o json + +title "a different key starts a new run" +trace $CLI air run -f run.yaml --idempotency-key other-key -o json | jq '.data.existing' + +title "a YAML idempotency_token is deduplicated the same way" +add_repl.py "$($CLI air run -f run-token.yaml -o json | jq -r .data.run_id)" YAML_TOKEN_RUN_ID +trace $CLI air run -f run-token.yaml diff --git a/acceptance/air/run-idempotency/test.toml b/acceptance/air/run-idempotency/test.toml new file mode 100644 index 00000000000..266538a8a61 --- /dev/null +++ b/acceptance/air/run-idempotency/test.toml @@ -0,0 +1,11 @@ +# Repeated submits against the fake workspace, whose runs/submit returns the run +# an already-used idempotency token started. + +# The SDK probes host reachability with a HEAD request; stub it for determinism. +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +[[Server]] +Pattern = "POST /api/2.0/ai-training/config:validate" +Response.Body = '{}' diff --git a/cmd/air/run.go b/cmd/air/run.go index 332f1aa1218..9009a2a02e9 100644 --- a/cmd/air/run.go +++ b/cmd/air/run.go @@ -33,6 +33,8 @@ type runResult struct { DryRun bool `json:"dry_run,omitempty"` RunID string `json:"run_id,omitempty"` DashboardURL string `json:"dashboard_url,omitempty"` + // Existing is nil for a dry run, which submits nothing. + Existing *bool `json:"existing,omitempty"` } const experimentalContainersWarning = `+--------------------------------------------------------------------+ @@ -187,24 +189,29 @@ The path must be a separate argument: cobra reserves -h as a boolean, so } w := cmdctx.WorkspaceClient(ctx) - runID, dashboardURL, err := submitWorkload(ctx, w, cfg, file, idempotencyKey, !jsonOut) + submitCtx, existing := withExistingRunReport(ctx) + runID, dashboardURL, err := submitWorkload(submitCtx, w, cfg, file, idempotencyKey, !jsonOut) if err != nil { return err } + existingKey := "" + if *existing { + existingKey = userIdempotencyToken(idempotencyKey, cfg) + } runIDStr := strconv.FormatInt(runID, 10) if !watch { if !jsonOut { out := cmd.OutOrStdout() - printSubmitResult(ctx, out, runIDStr, dashboardURL) + printSubmitResult(ctx, out, runIDStr, dashboardURL, existingKey) printPostSubmitGuidance(out, w.Config.Profile, runIDStr) grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, true) return nil } // PENDING is the submit status, distinct from the --watch JSONL // SUBMITTED event type below. - if err := renderEnvelope(ctx, runResult{Status: "PENDING", RunID: runIDStr, DashboardURL: dashboardURL}); err != nil { + if err := renderEnvelope(ctx, runResult{Status: "PENDING", RunID: runIDStr, DashboardURL: dashboardURL, Existing: existing}); err != nil { return err } grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, false) @@ -238,7 +245,7 @@ The path must be a separate argument: cobra reserves -h as a boolean, so monitoringMessage = fmt.Sprintf("Monitoring run and streaming logs from node 0 of %d...", cfg.Compute.NumAccelerators/perNode) } // The MLflow links stream in via the logs below, so don't poll here. - printSubmitResult(ctx, out, runIDStr, dashboardURL) + printSubmitResult(ctx, out, runIDStr, dashboardURL, existingKey) grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, true) // Separate the submit summary from the streamed logs. fmt.Fprintln(out) @@ -364,12 +371,17 @@ func handleWatchResult(out io.Writer, profile, runID string, err error) error { // printSubmitResult writes the green success line and Job Run link. The link is // styled (blue, underlined) and clickable, matching the `air get` view, and -// degrades to plain text on non-rich terminals. -func printSubmitResult(ctx context.Context, out io.Writer, runIDStr, dashboardURL string) { +// degrades to plain text on non-rich terminals. existingKey is the idempotency +// key when it returned an existing run, else empty. +func printSubmitResult(ctx context.Context, out io.Writer, runIDStr, dashboardURL, existingKey string) { renderer, colorOn := cmdio.NewRenderer(ctx, out) p := newPalette(renderer) - fmt.Fprintln(out, p.green.Render("Submitted workload with Job Run ID: "+runIDStr)) + line := "Submitted workload with Job Run ID: " + runIDStr + if existingKey != "" { + line = fmt.Sprintf("Returned existing workload with Job Run ID: %s (idempotency key %q was already used)", runIDStr, existingKey) + } + fmt.Fprintln(out, p.green.Render(line)) fmt.Fprintln(out, "View job run at: "+link(colorOn, p.blue, dashboardURL, dashboardURL)) } diff --git a/cmd/air/run_test.go b/cmd/air/run_test.go index 1e48b05a734..f26924f6319 100644 --- a/cmd/air/run_test.go +++ b/cmd/air/run_test.go @@ -128,10 +128,88 @@ func TestRunSubmitJSONStatusPending(t *testing.T) { out := buf.String() assert.Contains(t, out, `"status": "PENDING"`) assert.Contains(t, out, `"run_id": "555"`) + assert.Contains(t, out, `"existing": false`) // JSON stdout stays a clean envelope stream — no human-readable submit lines. assert.NotContains(t, out, "Submitting experiment") } +// idempotentSubmitServer runs the fake workspace, whose runs/submit returns the +// run a repeated idempotency token started. +func idempotentSubmitServer(t *testing.T) string { + t.Helper() + server := testserver.New(t) + t.Cleanup(server.Close) + server.Handle("POST", validateConfigPath, func(testserver.Request) any { + return validateConfigResponse{} + }) + testserver.AddDefaultHandlers(server) + return server.URL +} + +func runIdempotentSubmit(t *testing.T, srvURL string, out flags.Output, config string, args ...string) string { + t.Helper() + var buf bytes.Buffer + cfgPath := writeConfigFile(t, "run.yaml", config) + cmd := withOutput(newRunCommand(), out) + require.NoError(t, cmd.ParseFlags(append([]string{"--file", cfgPath}, args...))) + + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), out, nil, &buf, &buf, "", "")) + ctx = cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srvURL)) + cmd.SetContext(ctx) + cmd.SetOut(&buf) + require.NoError(t, cmd.RunE(cmd, nil)) + return buf.String() +} + +func TestRunSubmitIdempotencyKeyReturnsExistingRun(t *testing.T) { + srvURL := idempotentSubmitServer(t) + + first := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--idempotency-key", "flag-key") + assert.Contains(t, first, "Submitted workload with Job Run ID: ") + assert.NotContains(t, first, "existing") + + second := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--idempotency-key", "flag-key") + assert.Contains(t, second, "Returned existing workload with Job Run ID: ") + assert.Contains(t, second, `(idempotency key "flag-key" was already used)`) + assert.NotContains(t, second, "Submitted workload") + assert.Contains(t, second, "View job run at: ") + + other := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--idempotency-key", "other-key") + assert.Contains(t, other, "Submitted workload with Job Run ID: ") +} + +func TestRunSubmitYAMLIdempotencyTokenReturnsExistingRun(t *testing.T) { + srvURL := idempotentSubmitServer(t) + config := minimalConfig + "idempotency_token: yaml-key\n" + + first := runIdempotentSubmit(t, srvURL, flags.OutputText, config) + assert.Contains(t, first, "Submitted workload with Job Run ID: ") + + second := runIdempotentSubmit(t, srvURL, flags.OutputText, config) + assert.Contains(t, second, `(idempotency key "yaml-key" was already used)`) + assert.NotContains(t, second, "Submitted workload") +} + +func TestRunSubmitJSONReportsExistingRun(t *testing.T) { + srvURL := idempotentSubmitServer(t) + decode := func(out string) runResult { + var envelope struct { + Data runResult `json:"data"` + } + require.NoError(t, json.Unmarshal([]byte(out), &envelope), out) + require.NotNil(t, envelope.Data.Existing) + return envelope.Data + } + + first := decode(runIdempotentSubmit(t, srvURL, flags.OutputJSON, minimalConfig, "--idempotency-key", "json-key")) + assert.False(t, *first.Existing) + + second := decode(runIdempotentSubmit(t, srvURL, flags.OutputJSON, minimalConfig, "--idempotency-key", "json-key")) + assert.True(t, *second.Existing) + assert.Equal(t, first.RunID, second.RunID) + assert.Equal(t, "PENDING", second.Status) +} + type dryRunRequest struct { method string path string diff --git a/cmd/air/runsubmit.go b/cmd/air/runsubmit.go index f01198d2b7a..04bd5720ad6 100644 --- a/cmd/air/runsubmit.go +++ b/cmd/air/runsubmit.go @@ -17,6 +17,7 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/compute" @@ -267,19 +268,53 @@ func aiRuntimeTaskFromSubmitBody(body map[string]any) (map[string]any, error) { return aiRuntimeTask, nil } +// userIdempotencyToken returns the caller-supplied token: the --idempotency-key +// flag wins over the config's token. It is empty when neither is set. +func userIdempotencyToken(flag string, cfg *runConfig) string { + if flag == "" && cfg.IdempotencyToken != nil { + return *cfg.IdempotencyToken + } + return flag +} + // effectiveIdempotencyToken resolves the token sent to validation and submit: -// the --idempotency-key flag wins, then the config's token, else a generated one. +// the caller-supplied token, else a generated one. func effectiveIdempotencyToken(flag string, cfg *runConfig) string { - token := flag - if token == "" && cfg.IdempotencyToken != nil { - token = *cfg.IdempotencyToken - } + token := userIdempotencyToken(flag, cfg) if token == "" { token = uuid.New().String() } return token } +type existingRunKey struct{} + +// withExistingRunReport returns a context in which submitWorkload sets the +// returned bool when a caller-supplied idempotency token returned an existing +// run. Only the run command opts in, as the check costs an extra runs/get. +func withExistingRunReport(ctx context.Context) (context.Context, *bool) { + existing := new(bool) + return context.WithValue(ctx, existingRunKey{}, existing), existing +} + +// isExistingRun reports whether runID was started by an earlier submit that +// used the same idempotency token. runs/submit returns only a run id, so the +// run's command path is the signal: every submit uploads to a fresh launch +// directory, and a fresh run references this submit's commandPath while an +// existing run references an earlier one's. +func isExistingRun(ctx context.Context, w *databricks.WorkspaceClient, runID int64, commandPath string) bool { + run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: runID}) + if err != nil { + log.Warnf(ctx, "could not check whether run %d already existed: %v", runID, err) + return false + } + task := aiRuntimeTaskOf(run) + if task == nil || len(task.Deployments) == 0 || task.Deployments[0].CommandPath == "" { + return false + } + return task.Deployments[0].CommandPath != commandPath +} + // validateIdempotencyToken keeps the CLI's submission guard as a fallback when // backend validation is unavailable or does not yet enforce the same rule. func validateIdempotencyToken(token string) error { @@ -446,6 +481,12 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run return 0, "", err } + // A generated token is never a repeat. Container runs carry their command + // paths on containers, which the SDK's DeploymentSpec does not model. + if existing, ok := ctx.Value(existingRunKey{}).(*bool); ok && userIdempotencyToken(idempotencyKey, cfg) != "" && len(containers) == 0 { + *existing = isExistingRun(ctx, w, runID, commandPath) + } + dashboardURL = strings.TrimRight(w.Config.Host, "/") + "/jobs/runs/" + strconv.FormatInt(runID, 10) return runID, dashboardURL, nil } diff --git a/libs/testserver/jobs.go b/libs/testserver/jobs.go index 772d868cb14..8c4669c945f 100644 --- a/libs/testserver/jobs.go +++ b/libs/testserver/jobs.go @@ -647,6 +647,13 @@ func (s *FakeWorkspace) JobsSubmit(req Request) Response { defer s.LockUnlock()() + // A resent request with the same non-empty token returns the run that token started. + if request.IdempotencyToken != "" { + if runId, ok := s.JobRunIdempotency[request.IdempotencyToken]; ok { + return Response{Body: jobs.SubmitRunResponse{RunId: runId}} + } + } + runId := nextID() // The default run name for one-time runs is "Untitled" (Jobs API behavior). @@ -659,8 +666,9 @@ func (s *FakeWorkspace) JobsSubmit(req Request) Response { var tasks []jobs.RunTask for _, t := range request.Tasks { tasks = append(tasks, jobs.RunTask{ - RunId: nextID(), - TaskKey: t.TaskKey, + RunId: nextID(), + TaskKey: t.TaskKey, + AiRuntimeTask: t.AiRuntimeTask, State: &jobs.RunState{ LifeCycleState: jobs.RunLifeCycleStateRunning, }, @@ -679,6 +687,10 @@ func (s *FakeWorkspace) JobsSubmit(req Request) Response { Tasks: tasks, } + if request.IdempotencyToken != "" { + s.JobRunIdempotency[request.IdempotencyToken] = runId + } + // No tunnel server runs locally, so synthesize the metadata.json it would // publish; `ssh connect` polls for it before connecting. if strings.HasPrefix(runName, sshTunnelBootstrapRunPrefix) { diff --git a/libs/testserver/jobs_test.go b/libs/testserver/jobs_test.go index a5b04542deb..8228b6a7679 100644 --- a/libs/testserver/jobs_test.go +++ b/libs/testserver/jobs_test.go @@ -72,6 +72,31 @@ func TestJobsSubmit_DefaultsRunNameToUntitled(t *testing.T) { assert.Equal(t, "Untitled", run.RunName) } +func TestJobsSubmit_IdempotencyTokenReturnsExistingRun(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + task := func(commandPath string) []jobs.SubmitTask { + return []jobs.SubmitTask{{ + TaskKey: "main", + AiRuntimeTask: &jobs.AiRuntimeTask{ + Deployments: []jobs.DeploymentSpec{{CommandPath: commandPath}}, + }, + }} + } + + first := submitRun(t, workspace, jobs.SubmitRun{IdempotencyToken: "token", Tasks: task("/first/command.sh")}) + second := submitRun(t, workspace, jobs.SubmitRun{IdempotencyToken: "token", Tasks: task("/second/command.sh")}) + untokened := submitRun(t, workspace, jobs.SubmitRun{Tasks: task("/third/command.sh")}) + + assert.Equal(t, first.RunId, second.RunId) + assert.NotEqual(t, first.RunId, untokened.RunId) + + // The resent request must not overwrite the original run's spec. + run := getRun(t, workspace, first.RunId) + require.Len(t, run.Tasks, 1) + require.NotNil(t, run.Tasks[0].AiRuntimeTask) + assert.Equal(t, "/first/command.sh", run.Tasks[0].AiRuntimeTask.Deployments[0].CommandPath) +} + func TestJobsSubmit_RunReachesTerminalStateOnPoll(t *testing.T) { workspace := NewFakeWorkspace("http://test", "dbapi123")