diff --git a/.nextchanges/air/idempotency-existing-run.md b/.nextchanges/air/idempotency-existing-run.md new file mode 100644 index 00000000000..fb81df3ad13 --- /dev/null +++ b/.nextchanges/air/idempotency-existing-run.md @@ -0,0 +1 @@ +* `air run` now says when `--idempotency-key` or `idempotency_token` returned an existing run, and its `-o json` result reports this as `existing` (the `--watch` JSONL events do not). ([#7529](https://github.com/databricks/cli/pull/7529)) diff --git a/acceptance/air/run-idempotency/databricks.yml b/acceptance/air/run-idempotency/databricks.yml new file mode 100644 index 00000000000..a1c79b6244a --- /dev/null +++ b/acceptance/air/run-idempotency/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: air-run-idempotency diff --git a/acceptance/air/run-idempotency/out.test.toml b/acceptance/air/run-idempotency/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/air/run-idempotency/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/air/run-idempotency/output.txt b/acceptance/air/run-idempotency/output.txt new file mode 100644 index 00000000000..d53421f4022 --- /dev/null +++ b/acceptance/air/run-idempotency/output.txt @@ -0,0 +1,52 @@ + +=== first submit with a key starts a run + +>>> [CLI] air run -f run.yaml --idempotency-key flag-key +Submitting experiment: idempotency-smoke +Submitted workload with Job Run ID: [FLAG_KEY_RUN_ID] +View job run at: [DATABRICKS_URL]/jobs/runs/[FLAG_KEY_RUN_ID] + +Tip: use --watch when submitting a run to stream logs to your terminal. +Stream logs after submission using: + databricks air logs [FLAG_KEY_RUN_ID] + +=== resubmitting with the same key returns the existing run + +>>> [CLI] air run -f run.yaml --idempotency-key flag-key +Submitting experiment: idempotency-smoke +Returned existing workload with Job Run ID: [FLAG_KEY_RUN_ID] (idempotency key "flag-key" was already used) +View job run at: [DATABRICKS_URL]/jobs/runs/[FLAG_KEY_RUN_ID] + +Tip: use --watch when submitting a run to stream logs to your terminal. +Stream logs after submission using: + databricks air logs [FLAG_KEY_RUN_ID] + +=== JSON output reports the existing run + +>>> [CLI] air run -f run.yaml --idempotency-key flag-key -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "status": "PENDING", + "run_id": "[FLAG_KEY_RUN_ID]", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/[FLAG_KEY_RUN_ID]", + "existing": true + } +} + +=== a different key starts a new run + +>>> [CLI] air run -f run.yaml --idempotency-key other-key -o json +false + +=== a YAML idempotency_token is deduplicated the same way + +>>> [CLI] air run -f run-token.yaml +Submitting experiment: idempotency-smoke +Returned existing workload with Job Run ID: [YAML_TOKEN_RUN_ID] (idempotency key "yaml-key" was already used) +View job run at: [DATABRICKS_URL]/jobs/runs/[YAML_TOKEN_RUN_ID] + +Tip: use --watch when submitting a run to stream logs to your terminal. +Stream logs after submission using: + databricks air logs [YAML_TOKEN_RUN_ID] diff --git a/acceptance/air/run-idempotency/run-token.yaml b/acceptance/air/run-idempotency/run-token.yaml new file mode 100644 index 00000000000..63dd8d7b961 --- /dev/null +++ b/acceptance/air/run-idempotency/run-token.yaml @@ -0,0 +1,6 @@ +experiment_name: idempotency-smoke +command: python train.py +compute: + accelerator_type: GPU_1xH100 + num_accelerators: 1 +idempotency_token: yaml-key diff --git a/acceptance/air/run-idempotency/run.yaml b/acceptance/air/run-idempotency/run.yaml new file mode 100644 index 00000000000..38eecf38446 --- /dev/null +++ b/acceptance/air/run-idempotency/run.yaml @@ -0,0 +1,5 @@ +experiment_name: idempotency-smoke +command: python train.py +compute: + accelerator_type: GPU_1xH100 + num_accelerators: 1 diff --git a/acceptance/air/run-idempotency/script b/acceptance/air/run-idempotency/script new file mode 100644 index 00000000000..146e3d8521b --- /dev/null +++ b/acceptance/air/run-idempotency/script @@ -0,0 +1,16 @@ +title "first submit with a key starts a run" +trace $CLI air run -f run.yaml --idempotency-key flag-key +add_repl.py "$($CLI air run -f run.yaml --idempotency-key flag-key -o json | jq -r .data.run_id)" FLAG_KEY_RUN_ID + +title "resubmitting with the same key returns the existing run" +trace $CLI air run -f run.yaml --idempotency-key flag-key + +title "JSON output reports the existing run" +trace $CLI air run -f run.yaml --idempotency-key flag-key -o json + +title "a different key starts a new run" +trace $CLI air run -f run.yaml --idempotency-key other-key -o json | jq '.data.existing' + +title "a YAML idempotency_token is deduplicated the same way" +add_repl.py "$($CLI air run -f run-token.yaml -o json | jq -r .data.run_id)" YAML_TOKEN_RUN_ID +trace $CLI air run -f run-token.yaml diff --git a/acceptance/air/run-idempotency/test.toml b/acceptance/air/run-idempotency/test.toml new file mode 100644 index 00000000000..266538a8a61 --- /dev/null +++ b/acceptance/air/run-idempotency/test.toml @@ -0,0 +1,11 @@ +# Repeated submits against the fake workspace, whose runs/submit returns the run +# an already-used idempotency token started. + +# The SDK probes host reachability with a HEAD request; stub it for determinism. +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +[[Server]] +Pattern = "POST /api/2.0/ai-training/config:validate" +Response.Body = '{}' diff --git a/cmd/air/run.go b/cmd/air/run.go index 332f1aa1218..1c298051c0c 100644 --- a/cmd/air/run.go +++ b/cmd/air/run.go @@ -33,6 +33,9 @@ type runResult struct { DryRun bool `json:"dry_run,omitempty"` RunID string `json:"run_id,omitempty"` DashboardURL string `json:"dashboard_url,omitempty"` + // Existing is nil when unknown: a dry run submits nothing, and the + // existing-run lookup can fail or not apply (e.g. container runs). + Existing *bool `json:"existing,omitempty"` } const experimentalContainersWarning = `+--------------------------------------------------------------------+ @@ -187,24 +190,33 @@ The path must be a separate argument: cobra reserves -h as a boolean, so } w := cmdctx.WorkspaceClient(ctx) - runID, dashboardURL, err := submitWorkload(ctx, w, cfg, file, idempotencyKey, !jsonOut) + // The --watch JSON events don't report existing, so skip its runs/get there. + submitCtx, report := ctx, &existingRunReport{} + if !watch || !jsonOut { + submitCtx, report = withExistingRunReport(ctx) + } + runID, dashboardURL, err := submitWorkload(submitCtx, w, cfg, file, idempotencyKey, !jsonOut) if err != nil { return err } + existingKey := "" + if report.Existing != nil && *report.Existing { + existingKey = userIdempotencyToken(idempotencyKey, cfg) + } runIDStr := strconv.FormatInt(runID, 10) if !watch { if !jsonOut { out := cmd.OutOrStdout() - printSubmitResult(ctx, out, runIDStr, dashboardURL) + printSubmitResult(ctx, out, runIDStr, dashboardURL, existingKey) printPostSubmitGuidance(out, w.Config.Profile, runIDStr) grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, true) return nil } // PENDING is the submit status, distinct from the --watch JSONL // SUBMITTED event type below. - if err := renderEnvelope(ctx, runResult{Status: "PENDING", RunID: runIDStr, DashboardURL: dashboardURL}); err != nil { + if err := renderEnvelope(ctx, runResult{Status: "PENDING", RunID: runIDStr, DashboardURL: dashboardURL, Existing: report.Existing}); err != nil { return err } grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, false) @@ -238,7 +250,7 @@ The path must be a separate argument: cobra reserves -h as a boolean, so monitoringMessage = fmt.Sprintf("Monitoring run and streaming logs from node 0 of %d...", cfg.Compute.NumAccelerators/perNode) } // The MLflow links stream in via the logs below, so don't poll here. - printSubmitResult(ctx, out, runIDStr, dashboardURL) + printSubmitResult(ctx, out, runIDStr, dashboardURL, existingKey) grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, true) // Separate the submit summary from the streamed logs. fmt.Fprintln(out) @@ -364,12 +376,17 @@ func handleWatchResult(out io.Writer, profile, runID string, err error) error { // printSubmitResult writes the green success line and Job Run link. The link is // styled (blue, underlined) and clickable, matching the `air get` view, and -// degrades to plain text on non-rich terminals. -func printSubmitResult(ctx context.Context, out io.Writer, runIDStr, dashboardURL string) { +// degrades to plain text on non-rich terminals. existingKey is the idempotency +// key when it returned an existing run, else empty. +func printSubmitResult(ctx context.Context, out io.Writer, runIDStr, dashboardURL, existingKey string) { renderer, colorOn := cmdio.NewRenderer(ctx, out) p := newPalette(renderer) - fmt.Fprintln(out, p.green.Render("Submitted workload with Job Run ID: "+runIDStr)) + line := "Submitted workload with Job Run ID: " + runIDStr + if existingKey != "" { + line = fmt.Sprintf("Returned existing workload with Job Run ID: %s (idempotency key %q was already used)", runIDStr, existingKey) + } + fmt.Fprintln(out, p.green.Render(line)) fmt.Fprintln(out, "View job run at: "+link(colorOn, p.blue, dashboardURL, dashboardURL)) } diff --git a/cmd/air/run_test.go b/cmd/air/run_test.go index 1e48b05a734..0fe50b3336e 100644 --- a/cmd/air/run_test.go +++ b/cmd/air/run_test.go @@ -3,6 +3,7 @@ package aircmd import ( "bytes" "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" @@ -15,6 +16,7 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -128,10 +130,267 @@ func TestRunSubmitJSONStatusPending(t *testing.T) { out := buf.String() assert.Contains(t, out, `"status": "PENDING"`) assert.Contains(t, out, `"run_id": "555"`) + assert.Contains(t, out, `"existing": false`) // JSON stdout stays a clean envelope stream — no human-readable submit lines. assert.NotContains(t, out, "Submitting experiment") } +// idempotentSubmitServer runs the fake workspace, whose runs/submit returns the +// run a repeated idempotency token started. +func idempotentSubmitServer(t *testing.T) string { + t.Helper() + server := testserver.New(t) + t.Cleanup(server.Close) + server.Handle("POST", validateConfigPath, func(testserver.Request) any { + return validateConfigResponse{} + }) + testserver.AddDefaultHandlers(server) + return server.URL +} + +func runIdempotentSubmit(t *testing.T, srvURL string, out flags.Output, config string, args ...string) string { + t.Helper() + var buf bytes.Buffer + cfgPath := writeConfigFile(t, "run.yaml", config) + cmd := withOutput(newRunCommand(), out) + require.NoError(t, cmd.ParseFlags(append([]string{"--file", cfgPath}, args...))) + + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), out, nil, &buf, &buf, "", "")) + ctx = cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srvURL)) + cmd.SetContext(ctx) + cmd.SetOut(&buf) + require.NoError(t, cmd.RunE(cmd, nil)) + return buf.String() +} + +func TestRunSubmitIdempotencyKeyReturnsExistingRun(t *testing.T) { + srvURL := idempotentSubmitServer(t) + + first := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--idempotency-key", "flag-key") + assert.Contains(t, first, "Submitted workload with Job Run ID: ") + assert.NotContains(t, first, "existing") + + second := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--idempotency-key", "flag-key") + assert.Contains(t, second, "Returned existing workload with Job Run ID: ") + assert.Contains(t, second, `(idempotency key "flag-key" was already used)`) + assert.NotContains(t, second, "Submitted workload") + assert.Contains(t, second, "View job run at: ") + + other := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--idempotency-key", "other-key") + assert.Contains(t, other, "Submitted workload with Job Run ID: ") +} + +func TestRunSubmitYAMLIdempotencyTokenReturnsExistingRun(t *testing.T) { + srvURL := idempotentSubmitServer(t) + config := minimalConfig + "idempotency_token: yaml-key\n" + + first := runIdempotentSubmit(t, srvURL, flags.OutputText, config) + assert.Contains(t, first, "Submitted workload with Job Run ID: ") + + second := runIdempotentSubmit(t, srvURL, flags.OutputText, config) + assert.Contains(t, second, `(idempotency key "yaml-key" was already used)`) + assert.NotContains(t, second, "Submitted workload") +} + +func TestRunSubmitJSONReportsExistingRun(t *testing.T) { + srvURL := idempotentSubmitServer(t) + + first := decodeRunResult(t, runIdempotentSubmit(t, srvURL, flags.OutputJSON, minimalConfig, "--idempotency-key", "json-key")) + require.NotNil(t, first.Existing) + assert.False(t, *first.Existing) + + second := decodeRunResult(t, runIdempotentSubmit(t, srvURL, flags.OutputJSON, minimalConfig, "--idempotency-key", "json-key")) + require.NotNil(t, second.Existing) + assert.True(t, *second.Existing) + assert.Equal(t, first.RunID, second.RunID) + assert.Equal(t, "PENDING", second.Status) +} + +// earlierLaunchRunGet is a terminated run whose command path is from an earlier +// submit's launch directory, so a submit with a user key sees it as existing. +const earlierLaunchRunGet = `{ + "run_id": 777, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "tasks": [{ + "run_id": 778, + "attempt_number": 0, + "ai_runtime_task": {"deployments": [{"command_path": "/Users/u@example.com/.air/cli_launch/my-run/my-run_0123456789abcdef/command.sh"}]} + }] +}` + +// runGetReply answers runs/get; submittedCommandPath is the command path the +// preceding runs/submit sent. +type runGetReply func(w http.ResponseWriter, r *http.Request, submittedCommandPath string) + +func staticRunGet(status int, body string) runGetReply { + return func(w http.ResponseWriter, _ *http.Request, _ string) { + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + } +} + +// existingRunServer serves `air run` with or without --watch. runs/get replies +// via runGet; the returned counter tracks its calls. +func existingRunServer(t *testing.T, runGet runGetReply) (string, *atomic.Int32) { + t.Helper() + var runGets atomic.Int32 + var submittedCommandPath atomic.Value + submittedCommandPath.Store("") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/jobs/runs/submit"): + var req jobs.SubmitRun + if json.NewDecoder(r.Body).Decode(&req) == nil && len(req.Tasks) > 0 && req.Tasks[0].AiRuntimeTask != nil && len(req.Tasks[0].AiRuntimeTask.Deployments) > 0 { + submittedCommandPath.Store(req.Tasks[0].AiRuntimeTask.Deployments[0].CommandPath) + } + _, _ = w.Write([]byte(`{"run_id": 777}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get": + runGets.Add(1) + runGet(w, r, submittedCommandPath.Load().(string)) + case strings.HasPrefix(r.URL.Path, "/api/2.0/ai-training/workflows/by-run-id/"): + _, _ = w.Write([]byte(`{"log_records": [ + {"time_unix_nano": 1700000002000000000, "body": "step 2", "node_index": 0}, + {"time_unix_nano": 1700000001000000000, "body": "step 1", "node_index": 0} + ]}`)) + default: + _, _ = w.Write([]byte(`{"userName": "u@example.com", "workspace_id": 1}`)) + } + })) + t.Cleanup(srv.Close) + return srv.URL, &runGets +} + +// decodeRunResult returns the data of a non-watch `air run -o json` envelope. +func decodeRunResult(t *testing.T, out string) runResult { + t.Helper() + var envelope struct { + Data runResult `json:"data"` + } + require.NoError(t, json.Unmarshal([]byte(out), &envelope), out) + return envelope.Data +} + +func TestRunWatchExistingRunPrintsBeforeLogs(t *testing.T) { + srvURL, _ := existingRunServer(t, staticRunGet(http.StatusOK, earlierLaunchRunGet)) + + out := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--watch", "--idempotency-key", "watch-key") + existingLine := `Returned existing workload with Job Run ID: 777 (idempotency key "watch-key" was already used)` + require.Contains(t, out, existingLine) + require.Contains(t, out, "step 1\nstep 2") + assert.Less(t, strings.Index(out, existingLine), strings.Index(out, "step 1")) + assert.NotContains(t, out, "Submitted workload") +} + +func TestRunExistingRunLookupByOutputMode(t *testing.T) { + for _, tc := range []struct { + name string + out flags.Output + args []string + // wantLookups is the runs/get calls a user key adds over a generated token. + wantLookups int32 + }{ + {name: "text", out: flags.OutputText, wantLookups: 1}, + {name: "json", out: flags.OutputJSON, wantLookups: 1}, + {name: "watch text", out: flags.OutputText, args: []string{"--watch"}, wantLookups: 1}, + // The --watch JSON events don't report existing, so there is no lookup. + {name: "watch json", out: flags.OutputJSON, args: []string{"--watch"}, wantLookups: 0}, + } { + t.Run(tc.name, func(t *testing.T) { + generatedURL, generatedRunGets := existingRunServer(t, staticRunGet(http.StatusOK, earlierLaunchRunGet)) + runIdempotentSubmit(t, generatedURL, tc.out, minimalConfig, tc.args...) + + userKeyURL, userKeyRunGets := existingRunServer(t, staticRunGet(http.StatusOK, earlierLaunchRunGet)) + runIdempotentSubmit(t, userKeyURL, tc.out, minimalConfig, append(tc.args, "--idempotency-key", "mode-key")...) + + assert.Equal(t, tc.wantLookups, userKeyRunGets.Load()-generatedRunGets.Load()) + }) + } +} + +func TestRunSubmitExistingRunMatchesLaunchDirAcrossPathPrefixes(t *testing.T) { + // The backend echoes this submit's own command path in another prefix form. + srvURL, _ := existingRunServer(t, func(w http.ResponseWriter, _ *http.Request, submitted string) { + echoed := strings.TrimPrefix(submitted, "/Workspace") + require.NotEqual(t, submitted, echoed) + _, _ = fmt.Fprintf(w, `{"run_id": 777, "tasks": [{"run_id": 778, "ai_runtime_task": {"deployments": [{"command_path": %q}]}}]}`, echoed) + }) + + out := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--idempotency-key", "prefix-key") + assert.Contains(t, out, "Submitted workload with Job Run ID: 777") + + result := decodeRunResult(t, runIdempotentSubmit(t, srvURL, flags.OutputJSON, minimalConfig, "--idempotency-key", "prefix-key")) + require.NotNil(t, result.Existing) + assert.False(t, *result.Existing) +} + +func TestRunSubmitExistingRunLookupReportsUnknown(t *testing.T) { + for _, tc := range []struct { + name string + runGet runGetReply + }{ + { + name: "runs/get fails", + runGet: staticRunGet(http.StatusNotFound, `{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "run not found"}`), + }, + { + name: "runs/get times out", + runGet: func(_ http.ResponseWriter, r *http.Request, _ string) { + <-r.Context().Done() + }, + }, + { + name: "no deployment", + runGet: staticRunGet(http.StatusOK, `{"run_id": 777, "tasks": [{"run_id": 778, "ai_runtime_task": {}}]}`), + }, + { + name: "empty command path", + runGet: staticRunGet(http.StatusOK, `{"run_id": 777, "tasks": [{"run_id": 778, "ai_runtime_task": {"deployments": [{}]}}]}`), + }, + { + name: "unrecognized command path", + runGet: staticRunGet(http.StatusOK, `{"run_id": 777, "tasks": [{"run_id": 778, "ai_runtime_task": {"deployments": [{"command_path": "/Users/u@example.com/train.sh"}]}}]}`), + }, + } { + t.Run(tc.name, func(t *testing.T) { + defaultTimeout := existingRunLookupTimeout + existingRunLookupTimeout = 50 * time.Millisecond + t.Cleanup(func() { existingRunLookupTimeout = defaultTimeout }) + srvURL, runGets := existingRunServer(t, tc.runGet) + + out := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--idempotency-key", "unknown-key") + assert.Contains(t, out, "Submitted workload with Job Run ID: 777") + assert.NotContains(t, out, "Returned existing workload") + + jsonOut := runIdempotentSubmit(t, srvURL, flags.OutputJSON, minimalConfig, "--idempotency-key", "unknown-key") + assert.Nil(t, decodeRunResult(t, jsonOut).Existing) + assert.NotContains(t, jsonOut, `"existing"`) + assert.Equal(t, int32(2), runGets.Load()) + }) + } +} + +func TestRunSubmitContainersSkipsExistingRunLookup(t *testing.T) { + srvURL, runGets := existingRunServer(t, staticRunGet(http.StatusOK, earlierLaunchRunGet)) + config := ` +experiment_name: my-run +compute: + accelerator_type: GPU_1xH100 + num_accelerators: 1 +containers: + - name: inference + command: python infer.py + ranks: [0] + unity_catalog_image: main.ml.inference:v1 +` + + out := runIdempotentSubmit(t, srvURL, flags.OutputText, config, "--idempotency-key", "container-key") + assert.Contains(t, out, "Submitted workload with Job Run ID: 777") + + jsonOut := runIdempotentSubmit(t, srvURL, flags.OutputJSON, config, "--idempotency-key", "container-key") + assert.NotContains(t, jsonOut, `"existing"`) + assert.Zero(t, runGets.Load()) +} + type dryRunRequest struct { method string path string diff --git a/cmd/air/runlaunch.go b/cmd/air/runlaunch.go index a888d80d85d..c67345d8534 100644 --- a/cmd/air/runlaunch.go +++ b/cmd/air/runlaunch.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path" + "regexp" "strings" "uuid" @@ -58,6 +59,22 @@ func cliLaunchDir(base, experiment, run string) string { return path.Join(base, ".air", "cli_launch", experiment, run+"_"+unique) } +// launchDirPattern matches the _ directory name cliLaunchDir creates. +var launchDirPattern = regexp.MustCompile(`_[0-9a-f]{16}$`) + +// launchDirName returns the unique launch directory that holds commandPath's +// command script. Only this segment identifies a launch: the prefix can differ +// in form (/Users/... vs /Workspace/Users/...) between the submitted path and +// the one runs/get echoes. +func launchDirName(commandPath string) (string, bool) { + dir, file := path.Split(commandPath) + name := path.Base(dir) + if file != commandScriptName || !launchDirPattern.MatchString(name) { + return "", false + } + return name, true +} + // prospectiveLaunchPaths returns the workspace paths a submission would use // without creating directories or uploading artifacts. func prospectiveLaunchPaths(ctx context.Context, w *databricks.WorkspaceClient, cfg *runConfig) (string, string, string, error) { diff --git a/cmd/air/runsubmit.go b/cmd/air/runsubmit.go index f01198d2b7a..63df0f7dfd0 100644 --- a/cmd/air/runsubmit.go +++ b/cmd/air/runsubmit.go @@ -17,6 +17,7 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/compute" @@ -267,19 +268,72 @@ func aiRuntimeTaskFromSubmitBody(body map[string]any) (map[string]any, error) { return aiRuntimeTask, nil } +// userIdempotencyToken returns the caller-supplied token: the --idempotency-key +// flag wins over the config's token. It is empty when neither is set. +func userIdempotencyToken(flag string, cfg *runConfig) string { + if flag == "" && cfg.IdempotencyToken != nil { + return *cfg.IdempotencyToken + } + return flag +} + // effectiveIdempotencyToken resolves the token sent to validation and submit: -// the --idempotency-key flag wins, then the config's token, else a generated one. +// the caller-supplied token, else a generated one. func effectiveIdempotencyToken(flag string, cfg *runConfig) string { - token := flag - if token == "" && cfg.IdempotencyToken != nil { - token = *cfg.IdempotencyToken - } + token := userIdempotencyToken(flag, cfg) if token == "" { token = uuid.New().String() } return token } +type existingRunKey struct{} + +// existingRunReport records whether a submit returned an existing run. +// Existing is nil when the CLI could not tell. +type existingRunReport struct { + Existing *bool +} + +// withExistingRunReport returns a context in which submitWorkload fills the +// returned report. A context value avoids changing submitWorkload's signature +// for its many test callers, and lets callers opt in to the extra runs/get only +// where the result is shown. +func withExistingRunReport(ctx context.Context) (context.Context, *existingRunReport) { + report := &existingRunReport{} + return context.WithValue(ctx, existingRunKey{}, report), report +} + +// existingRunLookupTimeout bounds the runs/get behind the existing-run report: +// the report only refines the success line, so a slow backend must not hold +// it up. A var so tests can shorten it. +var existingRunLookupTimeout = 5 * time.Second + +// isExistingRun reports whether runID was started by an earlier submit that +// used the same idempotency token, or nil when that can't be determined. +// runs/submit returns only a run id, so the run's command path is the signal: +// every submit uploads to a fresh launch directory, and a fresh run references +// this submit's commandPath while an existing run references an earlier one's. +func isExistingRun(ctx context.Context, w *databricks.WorkspaceClient, runID int64, commandPath string) *bool { + ctx, cancel := context.WithTimeout(ctx, existingRunLookupTimeout) + defer cancel() + run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: runID}) + if err != nil { + log.Warnf(ctx, "could not check whether run %d already existed: %v", runID, err) + return nil + } + task := aiRuntimeTaskOf(run) + if task == nil || len(task.Deployments) == 0 { + return nil + } + got, gotOK := launchDirName(task.Deployments[0].CommandPath) + want, wantOK := launchDirName(commandPath) + if !gotOK || !wantOK { + return nil + } + return new(got != want) +} + // validateIdempotencyToken keeps the CLI's submission guard as a fallback when // backend validation is unavailable or does not yet enforce the same rule. func validateIdempotencyToken(token string) error { @@ -446,6 +500,18 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run return 0, "", err } + // A generated token is never a repeat, so that run is definitely new. + // Container runs stay unknown: they carry their command paths on containers, + // which the SDK's DeploymentSpec does not model. + if report, ok := ctx.Value(existingRunKey{}).(*existingRunReport); ok { + switch { + case userIdempotencyToken(idempotencyKey, cfg) == "": + report.Existing = new(false) + case len(containers) == 0: + report.Existing = isExistingRun(ctx, w, runID, commandPath) + } + } + dashboardURL = strings.TrimRight(w.Config.Host, "/") + "/jobs/runs/" + strconv.FormatInt(runID, 10) return runID, dashboardURL, nil } diff --git a/libs/testserver/jobs.go b/libs/testserver/jobs.go index 772d868cb14..20905dac80e 100644 --- a/libs/testserver/jobs.go +++ b/libs/testserver/jobs.go @@ -647,6 +647,14 @@ func (s *FakeWorkspace) JobsSubmit(req Request) Response { defer s.LockUnlock()() + // A resent request with the same non-empty token returns the run that token started. + // JobRunIdempotency is shared by runs/run-now and runs/submit. + if request.IdempotencyToken != "" { + if runId, ok := s.JobRunIdempotency[request.IdempotencyToken]; ok { + return Response{Body: jobs.SubmitRunResponse{RunId: runId}} + } + } + runId := nextID() // The default run name for one-time runs is "Untitled" (Jobs API behavior). @@ -659,8 +667,9 @@ func (s *FakeWorkspace) JobsSubmit(req Request) Response { var tasks []jobs.RunTask for _, t := range request.Tasks { tasks = append(tasks, jobs.RunTask{ - RunId: nextID(), - TaskKey: t.TaskKey, + RunId: nextID(), + TaskKey: t.TaskKey, + AiRuntimeTask: t.AiRuntimeTask, State: &jobs.RunState{ LifeCycleState: jobs.RunLifeCycleStateRunning, }, @@ -679,6 +688,10 @@ func (s *FakeWorkspace) JobsSubmit(req Request) Response { Tasks: tasks, } + if request.IdempotencyToken != "" { + s.JobRunIdempotency[request.IdempotencyToken] = runId + } + // No tunnel server runs locally, so synthesize the metadata.json it would // publish; `ssh connect` polls for it before connecting. if strings.HasPrefix(runName, sshTunnelBootstrapRunPrefix) { diff --git a/libs/testserver/jobs_test.go b/libs/testserver/jobs_test.go index a5b04542deb..8228b6a7679 100644 --- a/libs/testserver/jobs_test.go +++ b/libs/testserver/jobs_test.go @@ -72,6 +72,31 @@ func TestJobsSubmit_DefaultsRunNameToUntitled(t *testing.T) { assert.Equal(t, "Untitled", run.RunName) } +func TestJobsSubmit_IdempotencyTokenReturnsExistingRun(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + task := func(commandPath string) []jobs.SubmitTask { + return []jobs.SubmitTask{{ + TaskKey: "main", + AiRuntimeTask: &jobs.AiRuntimeTask{ + Deployments: []jobs.DeploymentSpec{{CommandPath: commandPath}}, + }, + }} + } + + first := submitRun(t, workspace, jobs.SubmitRun{IdempotencyToken: "token", Tasks: task("/first/command.sh")}) + second := submitRun(t, workspace, jobs.SubmitRun{IdempotencyToken: "token", Tasks: task("/second/command.sh")}) + untokened := submitRun(t, workspace, jobs.SubmitRun{Tasks: task("/third/command.sh")}) + + assert.Equal(t, first.RunId, second.RunId) + assert.NotEqual(t, first.RunId, untokened.RunId) + + // The resent request must not overwrite the original run's spec. + run := getRun(t, workspace, first.RunId) + require.Len(t, run.Tasks, 1) + require.NotNil(t, run.Tasks[0].AiRuntimeTask) + assert.Equal(t, "/first/command.sh", run.Tasks[0].AiRuntimeTask.Deployments[0].CommandPath) +} + func TestJobsSubmit_RunReachesTerminalStateOnPoll(t *testing.T) { workspace := NewFakeWorkspace("http://test", "dbapi123")