Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/7453.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `air get -o json` now reports `PENDING` while the run waits for compute, matching `air list`, and includes the run's `status_message`. ([#7453](https://github.com/databricks/cli/pull/7453))
5 changes: 3 additions & 2 deletions acceptance/air/get-ai-runtime/test.toml
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,12 @@ Response.Body = '''
}
'''

# MLflow identifiers for the deep-link, under ai_runtime_task_output.
# MLflow identifiers for the deep-link, under ai_runtime_task_output. The
# leftover status_message must not appear in the output of a finished run.
[[Server]]
Pattern = "GET /api/2.2/jobs/runs/get-output"
Response.Body = '''
{"ai_runtime_task_output": {"mlflow_experiment_id": "exp1", "mlflow_run_id": "run1"}}
{"ai_runtime_task_output": {"mlflow_experiment_id": "exp1", "mlflow_run_id": "run1", "status_message": "STATUS:Waiting for GPU compute capacity to become available."}}
'''

# The MLflow Run cell shows the run's name, fetched from the MLflow REST API.
Expand Down
1 change: 1 addition & 0 deletions acceptance/air/get-pending/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
19 changes: 19 additions & 0 deletions acceptance/air/get-pending/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@

>>> [CLI] air get 123 -o json
{
"v": 1,
"ts": "[TIMESTAMP]",
"data": {
"run_id": "123",
"status": "PENDING",
"started_at": null,
"duration_seconds": null,
"attempt_number": 0,
"experiment_name": "my-exp",
"dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]",
"mlflow_url": null,
"status_message": "Waiting for accelerator compute capacity to become available...",
"est_remaining_seconds": null,
"est_percent_complete": null
}
}
1 change: 1 addition & 0 deletions acceptance/air/get-pending/script
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
trace $CLI air get 123 -o json
42 changes: 42 additions & 0 deletions acceptance/air/get-pending/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# The SDK occasionally probes host reachability with a HEAD request; stub it so
# the test is deterministic.
[[Server]]
Pattern = "HEAD /"
Response.Body = ''

# The Jobs API reports the parent run as RUNNING while its task still waits for
# compute in the queue.
[[Server]]
Pattern = "GET /api/2.2/jobs/runs/get"
Response.Body = '''
{
"run_id": 123,
"run_page_url": "https://my-workspace.cloud.databricks.test/jobs/runs/123",
"creator_user_name": "user@example.com",
"state": {"life_cycle_state": "RUNNING"},
"tasks": [
{
"task_key": "train",
"run_id": 456,
"attempt_number": 0,
"max_retries": 0,
"state": {"life_cycle_state": "QUEUED"},
"ai_runtime_task": {
"experiment": "my-exp",
"deployments": [
{
"command_path": "/Workspace/Users/user@example.com/.air/cli_launch/my-exp/my-exp_abc/command.sh",
"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}
}
]
}
}
]
}
'''

[[Server]]
Pattern = "GET /api/2.2/jobs/runs/get-output"
Response.Body = '''
{"ai_runtime_task_output": {"status_message": "STATUS:Waiting for GPU compute capacity to become available."}}
'''
16 changes: 11 additions & 5 deletions cmd/air/get.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ type getData struct {
DashboardURL string `json:"dashboard_url"`
MLflowURL *string `json:"mlflow_url"`
TerminationReason *string `json:"termination_reason,omitempty"`
// StatusMessage is the progress message the text view shows as the status,
// set only while the run is unfinished.
StatusMessage *string `json:"status_message,omitempty"`
// ESTRemainingSeconds and ESTPercentComplete are a best-effort progress
// estimate for a running run, or null when one can't be made (see
// estimateTrainingETA).
Expand Down Expand Up @@ -181,6 +184,13 @@ JOB_RUN_ID is the job run ID returned by "air run".`,
}
}
}
statusMessage := ""
if taskOutput != nil && run.State != nil && run.State.ResultState == "" {
statusMessage = normalizeStatusMessage(taskOutput.StatusMessage)
}
if statusMessage != "" {
data.StatusMessage = &statusMessage
}
if task := findForEachTask(run); task != nil {
data.Sweep = buildSweepInfo(ctx, w, task)
} else if genAIComputeTask(run) == nil {
Expand All @@ -198,10 +208,6 @@ JOB_RUN_ID is the job run ID returned by "air run".`,
fmt.Fprintf(out, "Job Link: %s\n\n", hyperlink(ctx, out, data.DashboardURL, data.DashboardURL))
return renderEnvelope(ctx, data)
}
statusMessage := ""
if taskOutput != nil {
statusMessage = normalizeStatusMessage(taskOutput.StatusMessage)
}
data.DisplayStatus = detailedDisplayRunStatus(run, statusMessage)

renderRunText(ctx, out, w, run, &data, ids)
Expand Down Expand Up @@ -249,7 +255,7 @@ func aiRuntimeTaskOf(run *jobs.Run) *jobs.AiRuntimeTask {
func buildGetData(run *jobs.Run) getData {
data := getData{
RunID: strconv.FormatInt(run.RunId, 10),
Status: runStatus(run.State),
Status: displayRunStatus(run),
DisplayStatus: displayRunStatus(run),
TerminationReason: terminationReason(run),
StartedAt: startedAt(run),
Expand Down
17 changes: 17 additions & 0 deletions cmd/air/get_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,23 @@ func TestBuildGetDataTerminationReasonJSON(t *testing.T) {
assert.NotContains(t, string(body), "termination_reason")
}

func TestBuildGetDataWaitingForComputeStatus(t *testing.T) {
run := &jobs.Run{
RunId: 8,
State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning},
Tasks: []jobs.RunTask{{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateQueued}}},
}

body, err := json.Marshal(buildGetData(run))
require.NoError(t, err)

assert.JSONEq(t, `{
"run_id":"8","status":"PENDING","started_at":null,"duration_seconds":null,
"attempt_number":0,"experiment_name":null,"dashboard_url":"","mlflow_url":null,
"est_remaining_seconds":null,"est_percent_complete":null
}`, string(body))
}

func TestGetTemplateUsesDisplayStatus(t *testing.T) {
out := renderGet(t, getData{
RunID: "456",
Expand Down
Loading