Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/idempotency-existing-run.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `air run` now says when `--idempotency-key` or `idempotency_token` returned an existing run, and its `-o json` result reports this as `existing` (the `--watch` JSONL events do not). ([#7529](https://github.com/databricks/cli/pull/7529))
2 changes: 2 additions & 0 deletions acceptance/air/run-idempotency/databricks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
bundle:
name: air-run-idempotency
1 change: 1 addition & 0 deletions acceptance/air/run-idempotency/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
52 changes: 52 additions & 0 deletions acceptance/air/run-idempotency/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@

=== first submit with a key starts a run

>>> [CLI] air run -f run.yaml --idempotency-key flag-key
Submitting experiment: idempotency-smoke
Submitted workload with Job Run ID: [FLAG_KEY_RUN_ID]
View job run at: [DATABRICKS_URL]/jobs/runs/[FLAG_KEY_RUN_ID]

Tip: use --watch when submitting a run to stream logs to your terminal.
Stream logs after submission using:
databricks air logs [FLAG_KEY_RUN_ID]

=== resubmitting with the same key returns the existing run

>>> [CLI] air run -f run.yaml --idempotency-key flag-key
Submitting experiment: idempotency-smoke
Returned existing workload with Job Run ID: [FLAG_KEY_RUN_ID] (idempotency key "flag-key" was already used)
View job run at: [DATABRICKS_URL]/jobs/runs/[FLAG_KEY_RUN_ID]

Tip: use --watch when submitting a run to stream logs to your terminal.
Stream logs after submission using:
databricks air logs [FLAG_KEY_RUN_ID]

=== JSON output reports the existing run

>>> [CLI] air run -f run.yaml --idempotency-key flag-key -o json
{
"v": 1,
"ts": "[TIMESTAMP]",
"data": {
"status": "PENDING",
"run_id": "[FLAG_KEY_RUN_ID]",
"dashboard_url": "[DATABRICKS_URL]/jobs/runs/[FLAG_KEY_RUN_ID]",
"existing": true
}
}

=== a different key starts a new run

>>> [CLI] air run -f run.yaml --idempotency-key other-key -o json
false

=== a YAML idempotency_token is deduplicated the same way

>>> [CLI] air run -f run-token.yaml
Submitting experiment: idempotency-smoke
Returned existing workload with Job Run ID: [YAML_TOKEN_RUN_ID] (idempotency key "yaml-key" was already used)
View job run at: [DATABRICKS_URL]/jobs/runs/[YAML_TOKEN_RUN_ID]

Tip: use --watch when submitting a run to stream logs to your terminal.
Stream logs after submission using:
databricks air logs [YAML_TOKEN_RUN_ID]
6 changes: 6 additions & 0 deletions acceptance/air/run-idempotency/run-token.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
experiment_name: idempotency-smoke
command: python train.py
compute:
accelerator_type: GPU_1xH100
num_accelerators: 1
idempotency_token: yaml-key
5 changes: 5 additions & 0 deletions acceptance/air/run-idempotency/run.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
experiment_name: idempotency-smoke
command: python train.py
compute:
accelerator_type: GPU_1xH100
num_accelerators: 1
16 changes: 16 additions & 0 deletions acceptance/air/run-idempotency/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
title "first submit with a key starts a run"
trace $CLI air run -f run.yaml --idempotency-key flag-key
add_repl.py "$($CLI air run -f run.yaml --idempotency-key flag-key -o json | jq -r .data.run_id)" FLAG_KEY_RUN_ID

title "resubmitting with the same key returns the existing run"
trace $CLI air run -f run.yaml --idempotency-key flag-key

title "JSON output reports the existing run"
trace $CLI air run -f run.yaml --idempotency-key flag-key -o json

title "a different key starts a new run"
trace $CLI air run -f run.yaml --idempotency-key other-key -o json | jq '.data.existing'

title "a YAML idempotency_token is deduplicated the same way"
add_repl.py "$($CLI air run -f run-token.yaml -o json | jq -r .data.run_id)" YAML_TOKEN_RUN_ID
trace $CLI air run -f run-token.yaml
11 changes: 11 additions & 0 deletions acceptance/air/run-idempotency/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Repeated submits against the fake workspace, whose runs/submit returns the run
# an already-used idempotency token started.

# The SDK probes host reachability with a HEAD request; stub it for determinism.
[[Server]]
Pattern = "HEAD /"
Response.Body = ''

[[Server]]
Pattern = "POST /api/2.0/ai-training/config:validate"
Response.Body = '{}'
31 changes: 24 additions & 7 deletions cmd/air/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ type runResult struct {
DryRun bool `json:"dry_run,omitempty"`
RunID string `json:"run_id,omitempty"`
DashboardURL string `json:"dashboard_url,omitempty"`
// Existing is nil when unknown: a dry run submits nothing, and the
// existing-run lookup can fail or not apply (e.g. container runs).
Existing *bool `json:"existing,omitempty"`
}

const experimentalContainersWarning = `+--------------------------------------------------------------------+
Expand Down Expand Up @@ -187,24 +190,33 @@ The path must be a separate argument: cobra reserves -h as a boolean, so
}

w := cmdctx.WorkspaceClient(ctx)
runID, dashboardURL, err := submitWorkload(ctx, w, cfg, file, idempotencyKey, !jsonOut)
// The --watch JSON events don't report existing, so skip its runs/get there.
submitCtx, report := ctx, &existingRunReport{}
if !watch || !jsonOut {
submitCtx, report = withExistingRunReport(ctx)
}
runID, dashboardURL, err := submitWorkload(submitCtx, w, cfg, file, idempotencyKey, !jsonOut)
if err != nil {
return err
}
existingKey := ""
if report.Existing != nil && *report.Existing {
existingKey = userIdempotencyToken(idempotencyKey, cfg)
}

runIDStr := strconv.FormatInt(runID, 10)

if !watch {
if !jsonOut {
out := cmd.OutOrStdout()
printSubmitResult(ctx, out, runIDStr, dashboardURL)
printSubmitResult(ctx, out, runIDStr, dashboardURL, existingKey)
printPostSubmitGuidance(out, w.Config.Profile, runIDStr)
grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, true)
return nil
}
// PENDING is the submit status, distinct from the --watch JSONL
// SUBMITTED event type below.
if err := renderEnvelope(ctx, runResult{Status: "PENDING", RunID: runIDStr, DashboardURL: dashboardURL}); err != nil {
if err := renderEnvelope(ctx, runResult{Status: "PENDING", RunID: runIDStr, DashboardURL: dashboardURL, Existing: report.Existing}); err != nil {
return err
}
grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, false)
Expand Down Expand Up @@ -238,7 +250,7 @@ The path must be a separate argument: cobra reserves -h as a boolean, so
monitoringMessage = fmt.Sprintf("Monitoring run and streaming logs from node 0 of %d...", cfg.Compute.NumAccelerators/perNode)
}
// The MLflow links stream in via the logs below, so don't poll here.
printSubmitResult(ctx, out, runIDStr, dashboardURL)
printSubmitResult(ctx, out, runIDStr, dashboardURL, existingKey)
grantSubmittedPermissions(ctx, w, runID, cfg.Permissions, true)
// Separate the submit summary from the streamed logs.
fmt.Fprintln(out)
Expand Down Expand Up @@ -364,12 +376,17 @@ func handleWatchResult(out io.Writer, profile, runID string, err error) error {

// printSubmitResult writes the green success line and Job Run link. The link is
// styled (blue, underlined) and clickable, matching the `air get` view, and
// degrades to plain text on non-rich terminals.
func printSubmitResult(ctx context.Context, out io.Writer, runIDStr, dashboardURL string) {
// degrades to plain text on non-rich terminals. existingKey is the idempotency
// key when it returned an existing run, else empty.
func printSubmitResult(ctx context.Context, out io.Writer, runIDStr, dashboardURL, existingKey string) {
renderer, colorOn := cmdio.NewRenderer(ctx, out)
p := newPalette(renderer)

fmt.Fprintln(out, p.green.Render("Submitted workload with Job Run ID: "+runIDStr))
line := "Submitted workload with Job Run ID: " + runIDStr
if existingKey != "" {
line = fmt.Sprintf("Returned existing workload with Job Run ID: %s (idempotency key %q was already used)", runIDStr, existingKey)
}
fmt.Fprintln(out, p.green.Render(line))
fmt.Fprintln(out, "View job run at: "+link(colorOn, p.blue, dashboardURL, dashboardURL))
}

Expand Down
Loading
Loading