From 9ae49fbd149e052621d499e14964b93a15a3e404 Mon Sep 17 00:00:00 2001 From: Pardis Beikzadeh Date: Tue, 10 Nov 2026 21:35:47 +0000 Subject: [PATCH 1/2] air: Report pending compute status in `air get` JSON Co-authored-by: Isaac --- acceptance/air/get-ai-runtime/test.toml | 5 +-- acceptance/air/get-pending/out.test.toml | 1 + acceptance/air/get-pending/output.txt | 19 +++++++++++ acceptance/air/get-pending/script | 1 + acceptance/air/get-pending/test.toml | 42 ++++++++++++++++++++++++ cmd/air/get.go | 16 ++++++--- cmd/air/get_test.go | 17 ++++++++++ 7 files changed, 94 insertions(+), 7 deletions(-) create mode 100644 acceptance/air/get-pending/out.test.toml create mode 100644 acceptance/air/get-pending/output.txt create mode 100644 acceptance/air/get-pending/script create mode 100644 acceptance/air/get-pending/test.toml diff --git a/acceptance/air/get-ai-runtime/test.toml b/acceptance/air/get-ai-runtime/test.toml index 5aa909e6e1b..1879588b6c8 100644 --- a/acceptance/air/get-ai-runtime/test.toml +++ b/acceptance/air/get-ai-runtime/test.toml @@ -46,11 +46,12 @@ Response.Body = ''' } ''' -# MLflow identifiers for the deep-link, under ai_runtime_task_output. +# MLflow identifiers for the deep-link, under ai_runtime_task_output. The +# leftover status_message must not appear in the output of a finished run. [[Server]] Pattern = "GET /api/2.2/jobs/runs/get-output" Response.Body = ''' -{"ai_runtime_task_output": {"mlflow_experiment_id": "exp1", "mlflow_run_id": "run1"}} +{"ai_runtime_task_output": {"mlflow_experiment_id": "exp1", "mlflow_run_id": "run1", "status_message": "STATUS:Waiting for GPU compute capacity to become available."}} ''' # The MLflow Run cell shows the run's name, fetched from the MLflow REST API. diff --git a/acceptance/air/get-pending/out.test.toml b/acceptance/air/get-pending/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/air/get-pending/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/air/get-pending/output.txt b/acceptance/air/get-pending/output.txt new file mode 100644 index 00000000000..3a5f9a72eda --- /dev/null +++ b/acceptance/air/get-pending/output.txt @@ -0,0 +1,19 @@ + +>>> [CLI] air get 123 -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "run_id": "123", + "status": "PENDING", + "started_at": null, + "duration_seconds": null, + "attempt_number": 0, + "experiment_name": "my-exp", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", + "mlflow_url": null, + "status_message": "Waiting for accelerator compute capacity to become available...", + "est_remaining_seconds": null, + "est_percent_complete": null + } +} diff --git a/acceptance/air/get-pending/script b/acceptance/air/get-pending/script new file mode 100644 index 00000000000..5ef18da8e13 --- /dev/null +++ b/acceptance/air/get-pending/script @@ -0,0 +1 @@ +trace $CLI air get 123 -o json diff --git a/acceptance/air/get-pending/test.toml b/acceptance/air/get-pending/test.toml new file mode 100644 index 00000000000..3ff488ff00e --- /dev/null +++ b/acceptance/air/get-pending/test.toml @@ -0,0 +1,42 @@ +# The SDK occasionally probes host reachability with a HEAD request; stub it so +# the test is deterministic. +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +# The Jobs API reports the parent run as RUNNING while its task still waits for +# compute in the queue. +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get" +Response.Body = ''' +{ + "run_id": 123, + "run_page_url": "https://my-workspace.cloud.databricks.test/jobs/runs/123", + "creator_user_name": "user@example.com", + "state": {"life_cycle_state": "RUNNING"}, + "tasks": [ + { + "task_key": "train", + "run_id": 456, + "attempt_number": 0, + "max_retries": 0, + "state": {"life_cycle_state": "QUEUED"}, + "ai_runtime_task": { + "experiment": "my-exp", + "deployments": [ + { + "command_path": "/Workspace/Users/user@example.com/.air/cli_launch/my-exp/my-exp_abc/command.sh", + "compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1} + } + ] + } + } + ] +} +''' + +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get-output" +Response.Body = ''' +{"ai_runtime_task_output": {"status_message": "STATUS:Waiting for GPU compute capacity to become available."}} +''' diff --git a/cmd/air/get.go b/cmd/air/get.go index 9f55f0f60e8..ddb4b203bef 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -30,6 +30,9 @@ type getData struct { DashboardURL string `json:"dashboard_url"` MLflowURL *string `json:"mlflow_url"` TerminationReason *string `json:"termination_reason,omitempty"` + // StatusMessage is the progress message the text view shows as the status, + // set only while the run is unfinished. + StatusMessage *string `json:"status_message,omitempty"` // ESTRemainingSeconds and ESTPercentComplete are a best-effort progress // estimate for a running run, or null when one can't be made (see // estimateTrainingETA). @@ -181,6 +184,13 @@ JOB_RUN_ID is the job run ID returned by "air run".`, } } } + statusMessage := "" + if taskOutput != nil && run.State != nil && run.State.ResultState == "" { + statusMessage = normalizeStatusMessage(taskOutput.StatusMessage) + } + if statusMessage != "" { + data.StatusMessage = &statusMessage + } if task := findForEachTask(run); task != nil { data.Sweep = buildSweepInfo(ctx, w, task) } else if genAIComputeTask(run) == nil { @@ -198,10 +208,6 @@ JOB_RUN_ID is the job run ID returned by "air run".`, fmt.Fprintf(out, "Job Link: %s\n\n", hyperlink(ctx, out, data.DashboardURL, data.DashboardURL)) return renderEnvelope(ctx, data) } - statusMessage := "" - if taskOutput != nil { - statusMessage = normalizeStatusMessage(taskOutput.StatusMessage) - } data.DisplayStatus = detailedDisplayRunStatus(run, statusMessage) renderRunText(ctx, out, w, run, &data, ids) @@ -249,7 +255,7 @@ func aiRuntimeTaskOf(run *jobs.Run) *jobs.AiRuntimeTask { func buildGetData(run *jobs.Run) getData { data := getData{ RunID: strconv.FormatInt(run.RunId, 10), - Status: runStatus(run.State), + Status: displayRunStatus(run), DisplayStatus: displayRunStatus(run), TerminationReason: terminationReason(run), StartedAt: startedAt(run), diff --git a/cmd/air/get_test.go b/cmd/air/get_test.go index 7df2cf5e613..95c2d21519d 100644 --- a/cmd/air/get_test.go +++ b/cmd/air/get_test.go @@ -242,6 +242,23 @@ func TestBuildGetDataTerminationReasonJSON(t *testing.T) { assert.NotContains(t, string(body), "termination_reason") } +func TestBuildGetDataWaitingForComputeStatus(t *testing.T) { + run := &jobs.Run{ + RunId: 8, + State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, + Tasks: []jobs.RunTask{{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateQueued}}}, + } + + body, err := json.Marshal(buildGetData(run)) + require.NoError(t, err) + + assert.JSONEq(t, `{ + "run_id":"8","status":"PENDING","started_at":null,"duration_seconds":null, + "attempt_number":0,"experiment_name":null,"dashboard_url":"","mlflow_url":null, + "est_remaining_seconds":null,"est_percent_complete":null + }`, string(body)) +} + func TestGetTemplateUsesDisplayStatus(t *testing.T) { out := renderGet(t, getData{ RunID: "456", From 41fc017ec18e717a842beb611dd4046a5ffd6ce2 Mon Sep 17 00:00:00 2001 From: Pardis Beikzadeh Date: Tue, 10 Nov 2026 21:37:24 +0000 Subject: [PATCH 2/2] Add changelog entry Co-authored-by: Isaac --- .nextchanges/air/7453.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/air/7453.md diff --git a/.nextchanges/air/7453.md b/.nextchanges/air/7453.md new file mode 100644 index 00000000000..1b59dbb1a58 --- /dev/null +++ b/.nextchanges/air/7453.md @@ -0,0 +1 @@ +* `air get -o json` now reports `PENDING` while the run waits for compute, matching `air list`, and includes the run's `status_message`. ([#7453](https://github.com/databricks/cli/pull/7453))