Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/idempotency-existing-run.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `air run` now says when `--idempotency-key` or `idempotency_token` returned an existing run, and its JSON output reports this as `existing`. ([#7527](https://github.com/databricks/cli/pull/7527))
2 changes: 2 additions & 0 deletions acceptance/air/run-idempotency/databricks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
bundle:
name: air-run-idempotency
1 change: 1 addition & 0 deletions acceptance/air/run-idempotency/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
52 changes: 52 additions & 0 deletions acceptance/air/run-idempotency/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@

=== first submit with a key starts a run

>>> [CLI] air run -f run.yaml --idempotency-key flag-key
Submitting experiment: idempotency-smoke
Submitted workload with Job Run ID: [FLAG_KEY_RUN_ID]
View job run at: [DATABRICKS_URL]/jobs/runs/[FLAG_KEY_RUN_ID]

Tip: use --watch when submitting a run to stream logs to your terminal.
Stream logs after submission using:
databricks air logs [FLAG_KEY_RUN_ID]

=== resubmitting with the same key returns the existing run

>>> [CLI] air run -f run.yaml --idempotency-key flag-key
Submitting experiment: idempotency-smoke
Returned existing workload with Job Run ID: [FLAG_KEY_RUN_ID] (idempotency key "flag-key" was already used)
View job run at: [DATABRICKS_URL]/jobs/runs/[FLAG_KEY_RUN_ID]

Tip: use --watch when submitting a run to stream logs to your terminal.
Stream logs after submission using:
databricks air logs [FLAG_KEY_RUN_ID]

=== JSON output reports the existing run

>>> [CLI] air run -f run.yaml --idempotency-key flag-key -o json
{
"v": 1,
"ts": "[TIMESTAMP]",
"data": {
"status": "PENDING",
"run_id": "[FLAG_KEY_RUN_ID]",
"dashboard_url": "[DATABRICKS_URL]/jobs/runs/[FLAG_KEY_RUN_ID]",
"existing": true
}
}

=== a different key starts a new run

>>> [CLI] air run -f run.yaml --idempotency-key other-key -o json
false

=== a YAML idempotency_token is deduplicated the same way

>>> [CLI] air run -f run-token.yaml
Submitting experiment: idempotency-smoke
Returned existing workload with Job Run ID: [YAML_TOKEN_RUN_ID] (idempotency key "yaml-key" was already used)
View job run at: [DATABRICKS_URL]/jobs/runs/[YAML_TOKEN_RUN_ID]

Tip: use --watch when submitting a run to stream logs to your terminal.
Stream logs after submission using:
databricks air logs [YAML_TOKEN_RUN_ID]
6 changes: 6 additions & 0 deletions acceptance/air/run-idempotency/run-token.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
experiment_name: idempotency-smoke
command: python train.py
compute:
accelerator_type: GPU_1xH100
num_accelerators: 1
idempotency_token: yaml-key
5 changes: 5 additions & 0 deletions acceptance/air/run-idempotency/run.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
experiment_name: idempotency-smoke
command: python train.py
compute:
accelerator_type: GPU_1xH100
num_accelerators: 1
16 changes: 16 additions & 0 deletions acceptance/air/run-idempotency/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
title "first submit with a key starts a run"
trace $CLI air run -f run.yaml --idempotency-key flag-key
add_repl.py "$($CLI air run -f run.yaml --idempotency-key flag-key -o json | jq -r .data.run_id)" FLAG_KEY_RUN_ID

title "resubmitting with the same key returns the existing run"
trace $CLI air run -f run.yaml --idempotency-key flag-key

title "JSON output reports the existing run"
trace $CLI air run -f run.yaml --idempotency-key flag-key -o json

title "a different key starts a new run"
trace $CLI air run -f run.yaml --idempotency-key other-key -o json | jq '.data.existing'

title "a YAML idempotency_token is deduplicated the same way"
add_repl.py "$($CLI air run -f run-token.yaml -o json | jq -r .data.run_id)" YAML_TOKEN_RUN_ID
trace $CLI air run -f run-token.yaml
11 changes: 11 additions & 0 deletions acceptance/air/run-idempotency/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Repeated submits against the fake workspace, whose runs/submit returns the run
# an already-used idempotency token started.

# The SDK probes host reachability with a HEAD request; stub it for determinism.
[[Server]]
Pattern = "HEAD /"
Response.Body = ''

[[Server]]
Pattern = "POST /api/2.0/ai-training/config:validate"
Response.Body = '{}'
26 changes: 19 additions & 7 deletions cmd/air/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ type runResult struct {
DryRun bool `json:"dry_run,omitempty"`
RunID string `json:"run_id,omitempty"`
DashboardURL string `json:"dashboard_url,omitempty"`
// Existing is nil for a dry run, which submits nothing.
Existing *bool `json:"existing,omitempty"`
}

const experimentalContainersWarning = `+--------------------------------------------------------------------+
Expand Down Expand Up @@ -187,24 +189,29 @@ The path must be a separate argument: cobra reserves -h as a boolean, so
}

w := cmdctx.WorkspaceClient(ctx)
runID, dashboardURL, err := submitWorkload(ctx, w, cfg, file, idempotencyKey, !jsonOut)
submitCtx, existing := withExistingRunReport(ctx)
runID, dashboardURL, err := submitWorkload(submitCtx, w, cfg, file, idempotencyKey, !jsonOut)
if err != nil {
return err
}
existingKey := ""
if *existing {
existingKey = userIdempotencyToken(idempotencyKey, cfg)
}

runIDStr := strconv.FormatInt(runID, 10)

if !watch {
if !jsonOut {
out := cmd.OutOrStdout()
printSubmitResult(ctx, out, runIDStr, dashboardURL)
printSubmitResult(ctx, out, runIDStr, dashboardURL, existingKey)
printPostSubmitGuidance(out, w.Config.Profile, runIDStr)
grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, true)
return nil
}
// PENDING is the submit status, distinct from the --watch JSONL
// SUBMITTED event type below.
if err := renderEnvelope(ctx, runResult{Status: "PENDING", RunID: runIDStr, DashboardURL: dashboardURL}); err != nil {
if err := renderEnvelope(ctx, runResult{Status: "PENDING", RunID: runIDStr, DashboardURL: dashboardURL, Existing: existing}); err != nil {
return err
}
grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, false)
Expand Down Expand Up @@ -238,7 +245,7 @@ The path must be a separate argument: cobra reserves -h as a boolean, so
monitoringMessage = fmt.Sprintf("Monitoring run and streaming logs from node 0 of %d...", cfg.Compute.NumAccelerators/perNode)
}
// The MLflow links stream in via the logs below, so don't poll here.
printSubmitResult(ctx, out, runIDStr, dashboardURL)
printSubmitResult(ctx, out, runIDStr, dashboardURL, existingKey)
grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, true)
// Separate the submit summary from the streamed logs.
fmt.Fprintln(out)
Expand Down Expand Up @@ -364,12 +371,17 @@ func handleWatchResult(out io.Writer, profile, runID string, err error) error {

// printSubmitResult writes the green success line and Job Run link. The link is
// styled (blue, underlined) and clickable, matching the `air get` view, and
// degrades to plain text on non-rich terminals.
func printSubmitResult(ctx context.Context, out io.Writer, runIDStr, dashboardURL string) {
// degrades to plain text on non-rich terminals. existingKey is the idempotency
// key when it returned an existing run, else empty.
func printSubmitResult(ctx context.Context, out io.Writer, runIDStr, dashboardURL, existingKey string) {
renderer, colorOn := cmdio.NewRenderer(ctx, out)
p := newPalette(renderer)

fmt.Fprintln(out, p.green.Render("Submitted workload with Job Run ID: "+runIDStr))
line := "Submitted workload with Job Run ID: " + runIDStr
if existingKey != "" {
line = fmt.Sprintf("Returned existing workload with Job Run ID: %s (idempotency key %q was already used)", runIDStr, existingKey)
}
fmt.Fprintln(out, p.green.Render(line))
fmt.Fprintln(out, "View job run at: "+link(colorOn, p.blue, dashboardURL, dashboardURL))
}

Expand Down
78 changes: 78 additions & 0 deletions cmd/air/run_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,10 +128,88 @@ func TestRunSubmitJSONStatusPending(t *testing.T) {
out := buf.String()
assert.Contains(t, out, `"status": "PENDING"`)
assert.Contains(t, out, `"run_id": "555"`)
assert.Contains(t, out, `"existing": false`)
// JSON stdout stays a clean envelope stream — no human-readable submit lines.
assert.NotContains(t, out, "Submitting experiment")
}

// idempotentSubmitServer runs the fake workspace, whose runs/submit returns the
// run a repeated idempotency token started.
func idempotentSubmitServer(t *testing.T) string {
t.Helper()
server := testserver.New(t)
t.Cleanup(server.Close)
server.Handle("POST", validateConfigPath, func(testserver.Request) any {
return validateConfigResponse{}
})
testserver.AddDefaultHandlers(server)
return server.URL
}

func runIdempotentSubmit(t *testing.T, srvURL string, out flags.Output, config string, args ...string) string {
t.Helper()
var buf bytes.Buffer
cfgPath := writeConfigFile(t, "run.yaml", config)
cmd := withOutput(newRunCommand(), out)
require.NoError(t, cmd.ParseFlags(append([]string{"--file", cfgPath}, args...)))

ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), out, nil, &buf, &buf, "", ""))
ctx = cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srvURL))
cmd.SetContext(ctx)
cmd.SetOut(&buf)
require.NoError(t, cmd.RunE(cmd, nil))
return buf.String()
}

func TestRunSubmitIdempotencyKeyReturnsExistingRun(t *testing.T) {
srvURL := idempotentSubmitServer(t)

first := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--idempotency-key", "flag-key")
assert.Contains(t, first, "Submitted workload with Job Run ID: ")
assert.NotContains(t, first, "existing")

second := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--idempotency-key", "flag-key")
assert.Contains(t, second, "Returned existing workload with Job Run ID: ")
assert.Contains(t, second, `(idempotency key "flag-key" was already used)`)
assert.NotContains(t, second, "Submitted workload")
assert.Contains(t, second, "View job run at: ")

other := runIdempotentSubmit(t, srvURL, flags.OutputText, minimalConfig, "--idempotency-key", "other-key")
assert.Contains(t, other, "Submitted workload with Job Run ID: ")
}

func TestRunSubmitYAMLIdempotencyTokenReturnsExistingRun(t *testing.T) {
srvURL := idempotentSubmitServer(t)
config := minimalConfig + "idempotency_token: yaml-key\n"

first := runIdempotentSubmit(t, srvURL, flags.OutputText, config)
assert.Contains(t, first, "Submitted workload with Job Run ID: ")

second := runIdempotentSubmit(t, srvURL, flags.OutputText, config)
assert.Contains(t, second, `(idempotency key "yaml-key" was already used)`)
assert.NotContains(t, second, "Submitted workload")
}

func TestRunSubmitJSONReportsExistingRun(t *testing.T) {
srvURL := idempotentSubmitServer(t)
decode := func(out string) runResult {
var envelope struct {
Data runResult `json:"data"`
}
require.NoError(t, json.Unmarshal([]byte(out), &envelope), out)
require.NotNil(t, envelope.Data.Existing)
return envelope.Data
}

first := decode(runIdempotentSubmit(t, srvURL, flags.OutputJSON, minimalConfig, "--idempotency-key", "json-key"))
assert.False(t, *first.Existing)

second := decode(runIdempotentSubmit(t, srvURL, flags.OutputJSON, minimalConfig, "--idempotency-key", "json-key"))
assert.True(t, *second.Existing)
assert.Equal(t, first.RunID, second.RunID)
assert.Equal(t, "PENDING", second.Status)
}

type dryRunRequest struct {
method string
path string
Expand Down
51 changes: 46 additions & 5 deletions cmd/air/runsubmit.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/env"
"github.com/databricks/cli/libs/filer"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/client"
"github.com/databricks/databricks-sdk-go/service/compute"
Expand Down Expand Up @@ -267,19 +268,53 @@ func aiRuntimeTaskFromSubmitBody(body map[string]any) (map[string]any, error) {
return aiRuntimeTask, nil
}

// userIdempotencyToken returns the caller-supplied token: the --idempotency-key
// flag wins over the config's token. It is empty when neither is set.
func userIdempotencyToken(flag string, cfg *runConfig) string {
if flag == "" && cfg.IdempotencyToken != nil {
return *cfg.IdempotencyToken
}
return flag
}

// effectiveIdempotencyToken resolves the token sent to validation and submit:
// the --idempotency-key flag wins, then the config's token, else a generated one.
// the caller-supplied token, else a generated one.
func effectiveIdempotencyToken(flag string, cfg *runConfig) string {
token := flag
if token == "" && cfg.IdempotencyToken != nil {
token = *cfg.IdempotencyToken
}
token := userIdempotencyToken(flag, cfg)
if token == "" {
token = uuid.New().String()
}
return token
}

type existingRunKey struct{}

// withExistingRunReport returns a context in which submitWorkload sets the
// returned bool when a caller-supplied idempotency token returned an existing
// run. Only the run command opts in, as the check costs an extra runs/get.
func withExistingRunReport(ctx context.Context) (context.Context, *bool) {
existing := new(bool)
return context.WithValue(ctx, existingRunKey{}, existing), existing
}

// isExistingRun reports whether runID was started by an earlier submit that
// used the same idempotency token. runs/submit returns only a run id, so the
// run's command path is the signal: every submit uploads to a fresh launch
// directory, and a fresh run references this submit's commandPath while an
// existing run references an earlier one's.
func isExistingRun(ctx context.Context, w *databricks.WorkspaceClient, runID int64, commandPath string) bool {
run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: runID})
if err != nil {
log.Warnf(ctx, "could not check whether run %d already existed: %v", runID, err)
return false
}
task := aiRuntimeTaskOf(run)
if task == nil || len(task.Deployments) == 0 || task.Deployments[0].CommandPath == "" {
return false
}
return task.Deployments[0].CommandPath != commandPath
}

// validateIdempotencyToken keeps the CLI's submission guard as a fallback when
// backend validation is unavailable or does not yet enforce the same rule.
func validateIdempotencyToken(token string) error {
Expand Down Expand Up @@ -446,6 +481,12 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run
return 0, "", err
}

// A generated token is never a repeat. Container runs carry their command
// paths on containers, which the SDK's DeploymentSpec does not model.
if existing, ok := ctx.Value(existingRunKey{}).(*bool); ok && userIdempotencyToken(idempotencyKey, cfg) != "" && len(containers) == 0 {
*existing = isExistingRun(ctx, w, runID, commandPath)
}

dashboardURL = strings.TrimRight(w.Config.Host, "/") + "/jobs/runs/" + strconv.FormatInt(runID, 10)
return runID, dashboardURL, nil
}
Expand Down
16 changes: 14 additions & 2 deletions libs/testserver/jobs.go
Original file line number Diff line number Diff line change
Expand Up @@ -647,6 +647,13 @@ func (s *FakeWorkspace) JobsSubmit(req Request) Response {

defer s.LockUnlock()()

// A resent request with the same non-empty token returns the run that token started.
if request.IdempotencyToken != "" {
if runId, ok := s.JobRunIdempotency[request.IdempotencyToken]; ok {
return Response{Body: jobs.SubmitRunResponse{RunId: runId}}
}
}

runId := nextID()

// The default run name for one-time runs is "Untitled" (Jobs API behavior).
Expand All @@ -659,8 +666,9 @@ func (s *FakeWorkspace) JobsSubmit(req Request) Response {
var tasks []jobs.RunTask
for _, t := range request.Tasks {
tasks = append(tasks, jobs.RunTask{
RunId: nextID(),
TaskKey: t.TaskKey,
RunId: nextID(),
TaskKey: t.TaskKey,
AiRuntimeTask: t.AiRuntimeTask,
State: &jobs.RunState{
LifeCycleState: jobs.RunLifeCycleStateRunning,
},
Expand All @@ -679,6 +687,10 @@ func (s *FakeWorkspace) JobsSubmit(req Request) Response {
Tasks: tasks,
}

if request.IdempotencyToken != "" {
s.JobRunIdempotency[request.IdempotencyToken] = runId
}

// No tunnel server runs locally, so synthesize the metadata.json it would
// publish; `ssh connect` polls for it before connecting.
if strings.HasPrefix(runName, sshTunnelBootstrapRunPrefix) {
Expand Down
Loading
Loading