From 3d4078fc42eb0a2b3a7e77aa2490f16979d189cc Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Thu, 12 Nov 2026 23:11:57 +0000 Subject: [PATCH 1/2] air: Report the MLflow retry in air get -o json The retry fallback lands in mlflowIDsFromRetries, which sets OlderRetry when the MLflow identifiers come from an earlier retry (a last retry that failed before it started never got an MLflow run of its own). The text views show it as "(retry N)", but getData never output it, so `air get -o json` returned an earlier retry's MLflow IDs with no field saying which retry they came from. Add MLflowRetry (*int, json:"mlflow_retry,omitempty") to getData, set alongside mlflow_url. It is absent when the IDs come from the latest retry, so existing JSON output (and the acceptance goldens) is unchanged. Covered by a full-command JSON unit test for both shapes (older retry present with mlflow_retry: 0; latest attempt absent), and a new acceptance fixture pinning the no-MLflow-output envelope. Stacks on #7456. Co-authored-by: Isaac --- .nextchanges/air/air-get-json-mlflow-retry.md | 1 + acceptance/air/get-mlflow-retry/out.test.toml | 1 + acceptance/air/get-mlflow-retry/output.txt | 21 ++++ acceptance/air/get-mlflow-retry/script | 2 + acceptance/air/get-mlflow-retry/test.toml | 29 +++++ cmd/air/get.go | 20 ++-- cmd/air/get_test.go | 106 ++++++++++++++++++ 7 files changed, 172 insertions(+), 8 deletions(-) create mode 100644 .nextchanges/air/air-get-json-mlflow-retry.md create mode 100644 acceptance/air/get-mlflow-retry/out.test.toml create mode 100644 acceptance/air/get-mlflow-retry/output.txt create mode 100644 acceptance/air/get-mlflow-retry/script create mode 100644 acceptance/air/get-mlflow-retry/test.toml diff --git a/.nextchanges/air/air-get-json-mlflow-retry.md b/.nextchanges/air/air-get-json-mlflow-retry.md new file mode 100644 index 00000000000..a4b92651702 --- /dev/null +++ b/.nextchanges/air/air-get-json-mlflow-retry.md @@ -0,0 +1 @@ +* `air get -o json` now reports `mlflow_retry` when the MLflow link comes from an earlier retry because the run's last retry never reached MLflow. diff --git a/acceptance/air/get-mlflow-retry/out.test.toml b/acceptance/air/get-mlflow-retry/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/air/get-mlflow-retry/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/air/get-mlflow-retry/output.txt b/acceptance/air/get-mlflow-retry/output.txt new file mode 100644 index 00000000000..e43067e907a --- /dev/null +++ b/acceptance/air/get-mlflow-retry/output.txt @@ -0,0 +1,21 @@ + +=== get of a failed retry with no mlflow output (json) + +>>> [CLI] air get 123 -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "run_id": "123", + "status": "FAILED", + "started_at": "[TIMESTAMP]", + "duration_seconds": 12, + "attempt_number": 2, + "experiment_name": null, + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", + "mlflow_url": null, + "termination_reason": "Out of capacity", + "est_remaining_seconds": null, + "est_percent_complete": null + } +} diff --git a/acceptance/air/get-mlflow-retry/script b/acceptance/air/get-mlflow-retry/script new file mode 100644 index 00000000000..ac73ddfca05 --- /dev/null +++ b/acceptance/air/get-mlflow-retry/script @@ -0,0 +1,2 @@ +title "get of a failed retry with no mlflow output (json)" +errcode trace $CLI air get 123 -o json diff --git a/acceptance/air/get-mlflow-retry/test.toml b/acceptance/air/get-mlflow-retry/test.toml new file mode 100644 index 00000000000..5a92bea9868 --- /dev/null +++ b/acceptance/air/get-mlflow-retry/test.toml @@ -0,0 +1,29 @@ +# The SDK occasionally probes host reachability with a HEAD request; stub it so +# the test is deterministic. +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +# A failed run whose second and third attempts failed before they started, so +# neither ever logged an MLflow run of its own. +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get" +Response.Body = ''' +{ + "run_id": 123, + "start_time": 1700000000000, + "end_time": 1700000012000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "Out of capacity"}, + "tasks": [ + {"task_key": "train", "run_id": 456, "attempt_number": 0}, + {"task_key": "train", "run_id": 457, "attempt_number": 1}, + {"task_key": "train", "run_id": 458, "attempt_number": 2} + ] +} +''' + +# No attempt has MLflow output, so the JSON envelope must not invent an MLflow +# link or report an mlflow_retry. +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get-output" +Response.Body = '{}' diff --git a/cmd/air/get.go b/cmd/air/get.go index bac778ded51..348c3c01e20 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -23,14 +23,17 @@ import ( // the machine-readable output; fields tagged `json:"-"` are shown only in the // human-readable text view. type getData struct { - RunID string `json:"run_id"` - Status string `json:"status"` - StartedAt *string `json:"started_at"` - DurationSeconds *int64 `json:"duration_seconds"` - AttemptNumber int `json:"attempt_number"` - ExperimentName *string `json:"experiment_name"` - DashboardURL string `json:"dashboard_url"` - MLflowURL *string `json:"mlflow_url"` + RunID string `json:"run_id"` + Status string `json:"status"` + StartedAt *string `json:"started_at"` + DurationSeconds *int64 `json:"duration_seconds"` + AttemptNumber int `json:"attempt_number"` + ExperimentName *string `json:"experiment_name"` + DashboardURL string `json:"dashboard_url"` + MLflowURL *string `json:"mlflow_url"` + // MLflowRetry is the retry the MLflow IDs above came from when it isn't the + // run's latest (see mlflowIDsFromRetries), else absent. + MLflowRetry *int `json:"mlflow_retry,omitempty"` TerminationReason *string `json:"termination_reason,omitempty"` // StatusMessage is the progress message the text view shows as the status, // set only while the run is unfinished. @@ -186,6 +189,7 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, if ids != nil { url := mlflowLogsURL(w.Config.Host, ids) data.MLflowURL = &url + data.MLflowRetry = ids.OlderRetry // A remaining-time estimate only makes sense while the run is still // training; a terminal run is either done or stopped mid-progress. if isRunning(run) { diff --git a/cmd/air/get_test.go b/cmd/air/get_test.go index c6e984c3b44..73ce114a69a 100644 --- a/cmd/air/get_test.go +++ b/cmd/air/get_test.go @@ -199,6 +199,55 @@ func getRunServer(t *testing.T, meCalls *atomic.Int32) *httptest.Server { return srv } +// retryGetServer serves two runs for the full `air get` JSON flow: run 123, a +// failed two-attempt run whose last retry (458) never reached MLflow while +// attempt 0 (456) did, and run 124, a control whose only attempt (457) logged +// to MLflow. get-output is query-aware, so each retry's output is served +// separately. The Me endpoint also carries the workspace-ID header +// CurrentWorkspaceID reads. +func retryGetServer(t *testing.T) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/api/2.0/preview/scim/v2/Me": + w.Header().Set("X-Databricks-Org-Id", "9999") + _, _ = w.Write([]byte(`{"userName":"u@example.com"}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get" && r.URL.Query().Get("run_id") == "123": + _, _ = w.Write([]byte(`{ + "run_id": 123, + "start_time": 1700000000000, + "end_time": 1700000012000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED"}, + "tasks": [ + {"task_key": "train", "run_id": 456, "attempt_number": 0}, + {"task_key": "train", "run_id": 458, "attempt_number": 2} + ] + }`)) + case r.URL.Path == "/api/2.2/jobs/runs/get" && r.URL.Query().Get("run_id") == "124": + _, _ = w.Write([]byte(`{ + "run_id": 124, + "start_time": 1700000000000, + "end_time": 1700000012000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "tasks": [{"task_key": "train", "run_id": 457, "attempt_number": 0}] + }`)) + case r.URL.Path == "/api/2.2/jobs/runs/get-output": + switch r.URL.Query().Get("run_id") { + case "456": + _, _ = w.Write([]byte(`{"ai_runtime_task_output":{"mlflow_experiment_id":"E1","mlflow_run_id":"R0"}}`)) + case "457": + _, _ = w.Write([]byte(`{"ai_runtime_task_output":{"mlflow_experiment_id":"E1","mlflow_run_id":"R1"}}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + return srv +} + func runGetJSON(t *testing.T, w *databricks.WorkspaceClient) (string, error) { t.Helper() var buf bytes.Buffer @@ -247,6 +296,63 @@ func TestGetRunRejectsNonNumericWorkspaceID(t *testing.T) { assert.Equal(t, jsonError{Code: "INVALID_ARGS", Kind: "PERMANENT", Message: `configured workspace_id "not-a-number" is not a numeric workspace ID`}, got.Error) } +// getRunJSON runs `air get RUN_ID -o json` against srv and returns the decoded +// data payload. +func getRunJSON(t *testing.T, srv *httptest.Server, runID string) map[string]any { + t.Helper() + var buf bytes.Buffer + ctx := cmdctx.SetWorkspaceClient(t.Context(), newTestWorkspaceClient(t, srv.URL)) + ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputJSON, nil, &buf, &buf, "", "")) + cmd := withOutput(newGetCommand(), flags.OutputJSON) + cmd.SetContext(ctx) + require.NoError(t, cmd.RunE(cmd, []string{runID})) + + var got envelope + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + data, ok := got.Data.(map[string]any) + require.True(t, ok, "data payload should be an object") + return data +} + +// TestGetRunMLflowRetryJSON locks the JSON shape of the MLflow fields. A run +// whose last retry never reached MLflow links to an earlier retry's MLflow run +// and names that retry in mlflow_retry; a run whose latest attempt logged to +// MLflow links to it without the key. +func TestGetRunMLflowRetryJSON(t *testing.T) { + srv := retryGetServer(t) + host := srv.URL + + t.Run("an older retry's mlflow run names the retry", func(t *testing.T) { + data := getRunJSON(t, srv, "123") + assert.JSONEq(t, `{ + "run_id":"123","status":"FAILED","started_at":"2023-11-14T22:13:20+00:00", + "duration_seconds":12,"attempt_number":2,"experiment_name":null, + "dashboard_url":"`+host+`/jobs/runs/123?o=9999", + "mlflow_url":"`+host+`/ml/experiments/E1/runs/R0/logs", + "mlflow_retry":0, + "est_remaining_seconds":null,"est_percent_complete":null + }`, mustJSON(t, data)) + }) + + t.Run("the latest attempt's mlflow run has no mlflow_retry key", func(t *testing.T) { + data := getRunJSON(t, srv, "124") + assert.JSONEq(t, `{ + "run_id":"124","status":"SUCCESS","started_at":"2023-11-14T22:13:20+00:00", + "duration_seconds":12,"attempt_number":0,"experiment_name":null, + "dashboard_url":"`+host+`/jobs/runs/124?o=9999", + "mlflow_url":"`+host+`/ml/experiments/E1/runs/R1/logs", + "est_remaining_seconds":null,"est_percent_complete":null + }`, mustJSON(t, data)) + }) +} + +func mustJSON(t *testing.T, v any) string { + t.Helper() + body, err := json.Marshal(v) + require.NoError(t, err) + return string(body) +} + func TestGetTemplateSweep(t *testing.T) { out := renderGet(t, getData{ RunID: "456", From 8777f4bfd376bf1c664de6847b01e6cd1c23304a Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Fri, 13 Nov 2026 23:47:33 +0000 Subject: [PATCH 2/2] air: link the MLflow retry changelog fragment to the pull request Co-authored-by: Isaac --- .nextchanges/air/air-get-json-mlflow-retry.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/air/air-get-json-mlflow-retry.md b/.nextchanges/air/air-get-json-mlflow-retry.md index a4b92651702..2edd7c6ae90 100644 --- a/.nextchanges/air/air-get-json-mlflow-retry.md +++ b/.nextchanges/air/air-get-json-mlflow-retry.md @@ -1 +1 @@ -* `air get -o json` now reports `mlflow_retry` when the MLflow link comes from an earlier retry because the run's last retry never reached MLflow. +* `air get -o json` now reports `mlflow_retry` when the MLflow link comes from an earlier retry because the run's last retry never reached MLflow. ([#7620](https://github.com/databricks/cli/pull/7620))