diff --git a/.nextchanges/air/air-get-json-mlflow-retry.md b/.nextchanges/air/air-get-json-mlflow-retry.md new file mode 100644 index 00000000000..2edd7c6ae90 --- /dev/null +++ b/.nextchanges/air/air-get-json-mlflow-retry.md @@ -0,0 +1 @@ +* `air get -o json` now reports `mlflow_retry` when the MLflow link comes from an earlier retry because the run's last retry never reached MLflow. ([#7620](https://github.com/databricks/cli/pull/7620)) diff --git a/acceptance/air/get-mlflow-retry/out.test.toml b/acceptance/air/get-mlflow-retry/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/air/get-mlflow-retry/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/air/get-mlflow-retry/output.txt b/acceptance/air/get-mlflow-retry/output.txt new file mode 100644 index 00000000000..e43067e907a --- /dev/null +++ b/acceptance/air/get-mlflow-retry/output.txt @@ -0,0 +1,21 @@ + +=== get of a failed retry with no mlflow output (json) + +>>> [CLI] air get 123 -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "run_id": "123", + "status": "FAILED", + "started_at": "[TIMESTAMP]", + "duration_seconds": 12, + "attempt_number": 2, + "experiment_name": null, + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", + "mlflow_url": null, + "termination_reason": "Out of capacity", + "est_remaining_seconds": null, + "est_percent_complete": null + } +} diff --git a/acceptance/air/get-mlflow-retry/script b/acceptance/air/get-mlflow-retry/script new file mode 100644 index 00000000000..ac73ddfca05 --- /dev/null +++ b/acceptance/air/get-mlflow-retry/script @@ -0,0 +1,2 @@ +title "get of a failed retry with no mlflow output (json)" +errcode trace $CLI air get 123 -o json diff --git a/acceptance/air/get-mlflow-retry/test.toml b/acceptance/air/get-mlflow-retry/test.toml new file mode 100644 index 00000000000..5a92bea9868 --- /dev/null +++ b/acceptance/air/get-mlflow-retry/test.toml @@ -0,0 +1,29 @@ +# The SDK occasionally probes host reachability with a HEAD request; stub it so +# the test is deterministic. +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +# A failed run whose second and third attempts failed before they started, so +# neither ever logged an MLflow run of its own. +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get" +Response.Body = ''' +{ + "run_id": 123, + "start_time": 1700000000000, + "end_time": 1700000012000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "Out of capacity"}, + "tasks": [ + {"task_key": "train", "run_id": 456, "attempt_number": 0}, + {"task_key": "train", "run_id": 457, "attempt_number": 1}, + {"task_key": "train", "run_id": 458, "attempt_number": 2} + ] +} +''' + +# No attempt has MLflow output, so the JSON envelope must not invent an MLflow +# link or report an mlflow_retry. +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get-output" +Response.Body = '{}' diff --git a/cmd/air/get.go b/cmd/air/get.go index bac778ded51..348c3c01e20 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -23,14 +23,17 @@ import ( // the machine-readable output; fields tagged `json:"-"` are shown only in the // human-readable text view. type getData struct { - RunID string `json:"run_id"` - Status string `json:"status"` - StartedAt *string `json:"started_at"` - DurationSeconds *int64 `json:"duration_seconds"` - AttemptNumber int `json:"attempt_number"` - ExperimentName *string `json:"experiment_name"` - DashboardURL string `json:"dashboard_url"` - MLflowURL *string `json:"mlflow_url"` + RunID string `json:"run_id"` + Status string `json:"status"` + StartedAt *string `json:"started_at"` + DurationSeconds *int64 `json:"duration_seconds"` + AttemptNumber int `json:"attempt_number"` + ExperimentName *string `json:"experiment_name"` + DashboardURL string `json:"dashboard_url"` + MLflowURL *string `json:"mlflow_url"` + // MLflowRetry is the retry the MLflow IDs above came from when it isn't the + // run's latest (see mlflowIDsFromRetries), else absent. + MLflowRetry *int `json:"mlflow_retry,omitempty"` TerminationReason *string `json:"termination_reason,omitempty"` // StatusMessage is the progress message the text view shows as the status, // set only while the run is unfinished. @@ -186,6 +189,7 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, if ids != nil { url := mlflowLogsURL(w.Config.Host, ids) data.MLflowURL = &url + data.MLflowRetry = ids.OlderRetry // A remaining-time estimate only makes sense while the run is still // training; a terminal run is either done or stopped mid-progress. if isRunning(run) { diff --git a/cmd/air/get_test.go b/cmd/air/get_test.go index c6e984c3b44..73ce114a69a 100644 --- a/cmd/air/get_test.go +++ b/cmd/air/get_test.go @@ -199,6 +199,55 @@ func getRunServer(t *testing.T, meCalls *atomic.Int32) *httptest.Server { return srv } +// retryGetServer serves two runs for the full `air get` JSON flow: run 123, a +// failed two-attempt run whose last retry (458) never reached MLflow while +// attempt 0 (456) did, and run 124, a control whose only attempt (457) logged +// to MLflow. get-output is query-aware, so each retry's output is served +// separately. The Me endpoint also carries the workspace-ID header +// CurrentWorkspaceID reads. +func retryGetServer(t *testing.T) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/api/2.0/preview/scim/v2/Me": + w.Header().Set("X-Databricks-Org-Id", "9999") + _, _ = w.Write([]byte(`{"userName":"u@example.com"}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get" && r.URL.Query().Get("run_id") == "123": + _, _ = w.Write([]byte(`{ + "run_id": 123, + "start_time": 1700000000000, + "end_time": 1700000012000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED"}, + "tasks": [ + {"task_key": "train", "run_id": 456, "attempt_number": 0}, + {"task_key": "train", "run_id": 458, "attempt_number": 2} + ] + }`)) + case r.URL.Path == "/api/2.2/jobs/runs/get" && r.URL.Query().Get("run_id") == "124": + _, _ = w.Write([]byte(`{ + "run_id": 124, + "start_time": 1700000000000, + "end_time": 1700000012000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "tasks": [{"task_key": "train", "run_id": 457, "attempt_number": 0}] + }`)) + case r.URL.Path == "/api/2.2/jobs/runs/get-output": + switch r.URL.Query().Get("run_id") { + case "456": + _, _ = w.Write([]byte(`{"ai_runtime_task_output":{"mlflow_experiment_id":"E1","mlflow_run_id":"R0"}}`)) + case "457": + _, _ = w.Write([]byte(`{"ai_runtime_task_output":{"mlflow_experiment_id":"E1","mlflow_run_id":"R1"}}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + return srv +} + func runGetJSON(t *testing.T, w *databricks.WorkspaceClient) (string, error) { t.Helper() var buf bytes.Buffer @@ -247,6 +296,63 @@ func TestGetRunRejectsNonNumericWorkspaceID(t *testing.T) { assert.Equal(t, jsonError{Code: "INVALID_ARGS", Kind: "PERMANENT", Message: `configured workspace_id "not-a-number" is not a numeric workspace ID`}, got.Error) } +// getRunJSON runs `air get RUN_ID -o json` against srv and returns the decoded +// data payload. +func getRunJSON(t *testing.T, srv *httptest.Server, runID string) map[string]any { + t.Helper() + var buf bytes.Buffer + ctx := cmdctx.SetWorkspaceClient(t.Context(), newTestWorkspaceClient(t, srv.URL)) + ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputJSON, nil, &buf, &buf, "", "")) + cmd := withOutput(newGetCommand(), flags.OutputJSON) + cmd.SetContext(ctx) + require.NoError(t, cmd.RunE(cmd, []string{runID})) + + var got envelope + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + data, ok := got.Data.(map[string]any) + require.True(t, ok, "data payload should be an object") + return data +} + +// TestGetRunMLflowRetryJSON locks the JSON shape of the MLflow fields. A run +// whose last retry never reached MLflow links to an earlier retry's MLflow run +// and names that retry in mlflow_retry; a run whose latest attempt logged to +// MLflow links to it without the key. +func TestGetRunMLflowRetryJSON(t *testing.T) { + srv := retryGetServer(t) + host := srv.URL + + t.Run("an older retry's mlflow run names the retry", func(t *testing.T) { + data := getRunJSON(t, srv, "123") + assert.JSONEq(t, `{ + "run_id":"123","status":"FAILED","started_at":"2023-11-14T22:13:20+00:00", + "duration_seconds":12,"attempt_number":2,"experiment_name":null, + "dashboard_url":"`+host+`/jobs/runs/123?o=9999", + "mlflow_url":"`+host+`/ml/experiments/E1/runs/R0/logs", + "mlflow_retry":0, + "est_remaining_seconds":null,"est_percent_complete":null + }`, mustJSON(t, data)) + }) + + t.Run("the latest attempt's mlflow run has no mlflow_retry key", func(t *testing.T) { + data := getRunJSON(t, srv, "124") + assert.JSONEq(t, `{ + "run_id":"124","status":"SUCCESS","started_at":"2023-11-14T22:13:20+00:00", + "duration_seconds":12,"attempt_number":0,"experiment_name":null, + "dashboard_url":"`+host+`/jobs/runs/124?o=9999", + "mlflow_url":"`+host+`/ml/experiments/E1/runs/R1/logs", + "est_remaining_seconds":null,"est_percent_complete":null + }`, mustJSON(t, data)) + }) +} + +func mustJSON(t *testing.T, v any) string { + t.Helper() + body, err := json.Marshal(v) + require.NoError(t, err) + return string(body) +} + func TestGetTemplateSweep(t *testing.T) { out := renderGet(t, getData{ RunID: "456",