From 76e217fee23c1304803e3fa84012fb2bb6704b90 Mon Sep 17 00:00:00 2001 From: Pardis Beikzadeh Date: Tue, 10 Nov 2026 23:27:53 +0000 Subject: [PATCH 01/10] air: Show full failure reasons and MLflow links in air list panes Co-authored-by: Isaac --- .nextchanges/air/air-list-detail-panes.md | 1 + .../air/logs-failed-retry/out.test.toml | 1 + acceptance/air/logs-failed-retry/output.txt | 6 +++ acceptance/air/logs-failed-retry/script | 2 + acceptance/air/logs-failed-retry/test.toml | 30 +++++++++++++++ cmd/air/get.go | 2 +- cmd/air/list_detail.go | 8 +++- cmd/air/list_detail_test.go | 37 +++++++++++++++++++ cmd/air/list_tui.go | 11 +++++- cmd/air/list_tui_test.go | 21 +++++++++++ cmd/air/logstream_support.go | 5 +++ cmd/air/logstream_test.go | 20 +++++++++- cmd/air/mlflow.go | 24 ++++++++++-- cmd/air/mlflow_test.go | 25 +++++++++++-- 14 files changed, 180 insertions(+), 13 deletions(-) create mode 100644 .nextchanges/air/air-list-detail-panes.md create mode 100644 acceptance/air/logs-failed-retry/out.test.toml create mode 100644 acceptance/air/logs-failed-retry/output.txt create mode 100644 acceptance/air/logs-failed-retry/script create mode 100644 acceptance/air/logs-failed-retry/test.toml create mode 100644 cmd/air/list_detail_test.go diff --git a/.nextchanges/air/air-list-detail-panes.md b/.nextchanges/air/air-list-detail-panes.md new file mode 100644 index 00000000000..26f1e2fb4fd --- /dev/null +++ b/.nextchanges/air/air-list-detail-panes.md @@ -0,0 +1 @@ +* Wrap long failure reasons and log lines in the `air list` info and logs panes, keep the MLflow link when a run's last retry failed before starting, and point `air logs` at earlier retries when the last one has no logs. diff --git a/acceptance/air/logs-failed-retry/out.test.toml b/acceptance/air/logs-failed-retry/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/air/logs-failed-retry/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/air/logs-failed-retry/output.txt b/acceptance/air/logs-failed-retry/output.txt new file mode 100644 index 00000000000..7bdcc1c0562 --- /dev/null +++ b/acceptance/air/logs-failed-retry/output.txt @@ -0,0 +1,6 @@ + +=== logs of a failed retry with no logs point to the earlier retry +>>> [CLI] air logs 123 +No logs available for run 123. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs: databricks air logs 123 --retry 0 + +Exit code: 1 diff --git a/acceptance/air/logs-failed-retry/script b/acceptance/air/logs-failed-retry/script new file mode 100644 index 00000000000..741e68c28b2 --- /dev/null +++ b/acceptance/air/logs-failed-retry/script @@ -0,0 +1,2 @@ +title "logs of a failed retry with no logs point to the earlier retry" +errcode trace $CLI air logs 123 diff --git a/acceptance/air/logs-failed-retry/test.toml b/acceptance/air/logs-failed-retry/test.toml new file mode 100644 index 00000000000..14224ee8f14 --- /dev/null +++ b/acceptance/air/logs-failed-retry/test.toml @@ -0,0 +1,30 @@ +# The SDK occasionally probes host reachability with a HEAD request; stub it so +# the test is deterministic. +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +# A failed run whose second attempt failed before it started. +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get" +Response.Body = ''' +{ + "run_id": 123, + "start_time": 1700000000000, + "end_time": 1700000012000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "Out of capacity"}, + "tasks": [ + {"task_key": "train", "run_id": 456, "attempt_number": 0}, + {"task_key": "train", "run_id": 457, "attempt_number": 1} + ] +} +''' + +# Neither Bricklens nor MLflow has logs for the latest attempt. +[[Server]] +Pattern = "GET /api/2.0/ai-training/workflows/by-run-id/123/logs" +Response.Body = '{"log_records": []}' + +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get-output" +Response.Body = '{}' diff --git a/cmd/air/get.go b/cmd/air/get.go index 9f55f0f60e8..6466aab7a20 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -165,7 +165,7 @@ JOB_RUN_ID is the job run ID returned by "air run".`, data := buildGetData(run) data.DashboardURL = dashboardURL(w.Config.Host, runID, workspaceID) taskOutput := aiRuntimeTaskOutput(ctx, w, run) - ids := mlflowIDsFromOutput(taskOutput) + ids := mlflowIDsFromAttempts(ctx, w, run, taskOutput) if ids != nil { url := mlflowLogsURL(w.Config.Host, ids) data.MLflowURL = &url diff --git a/cmd/air/list_detail.go b/cmd/air/list_detail.go index c0eca25b13f..62627030605 100644 --- a/cmd/air/list_detail.go +++ b/cmd/air/list_detail.go @@ -39,15 +39,16 @@ func runDetailText(ctx context.Context, w *databricks.WorkspaceClient, runID int } // runLogsSnapshot fetches a one-shot tail of a run's logs into a string, for the -// list picker's detail pane. +// list picker's detail pane, followed by the run's MLflow logs link. func runLogsSnapshot(ctx context.Context, w *databricks.WorkspaceClient, runID int64) (string, error) { - status, err := resolveRunStatus(ctx, w, runID) + run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: runID}) if err != nil { if errors.Is(err, apierr.ErrResourceDoesNotExist) { return "", fmt.Errorf("run %d not found", runID) } return "", fmt.Errorf("failed to fetch run status: %w", err) } + status := projectRunStatus(run) req := logRequest{ runID: runID, @@ -60,5 +61,8 @@ func runLogsSnapshot(ctx context.Context, w *databricks.WorkspaceClient, runID i if _, err := fetchLogs(ctx, w, &buf, req, status); err != nil { return "", fmt.Errorf("failed to fetch logs: %w", err) } + if ids := mlflowIDs(ctx, w, run); ids != nil { + fmt.Fprintf(&buf, "\nMLflow URL: %s\n", mlflowLogsURL(w.Config.Host, ids)) + } return buf.String(), nil } diff --git a/cmd/air/list_detail_test.go b/cmd/air/list_detail_test.go new file mode 100644 index 00000000000..b3d2cde9df5 --- /dev/null +++ b/cmd/air/list_detail_test.go @@ -0,0 +1,37 @@ +package aircmd + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRunLogsSnapshotFailedRetryWithoutLogs(t *testing.T) { + // The last retry failed before it started: no logs and no MLflow output of + // its own. The pane explains that and still links the MLflow run that + // attempt 0 logged to. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/logs"): + _, _ = w.Write([]byte(`{"log_records": []}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get": + _, _ = w.Write([]byte(`{"run_id": 123, "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "Out of capacity"}, + "tasks": [{"run_id": 456, "attempt_number": 0}, {"run_id": 457, "attempt_number": 1}]}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get-output" && r.URL.Query().Get("run_id") == "456": + _, _ = w.Write([]byte(`{"ai_runtime_task_output": {"mlflow_experiment_id": "E1", "mlflow_run_id": "R1"}}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + + got, err := runLogsSnapshot(t.Context(), newTestWorkspaceClient(t, srv.URL), 123) + require.NoError(t, err) + assert.Equal(t, "No logs available for run 123. Run terminated in state FAILED: Out of capacity. "+ + "Earlier retries may have logs: databricks air logs 123 --retry 0\n"+ + "\nMLflow URL: "+srv.URL+"/ml/experiments/E1/runs/R1/artifacts/logs/node_0\n", got) +} diff --git a/cmd/air/list_tui.go b/cmd/air/list_tui.go index 10ce1ca76dc..ea2264e8ad0 100644 --- a/cmd/air/list_tui.go +++ b/cmd/air/list_tui.go @@ -9,6 +9,7 @@ import ( "github.com/charmbracelet/bubbles/viewport" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" + "github.com/charmbracelet/x/ansi" "github.com/databricks/cli/libs/browser" "github.com/databricks/cli/libs/cmdio" "github.com/spf13/cobra" @@ -184,6 +185,7 @@ func (m listModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.height = msg.Height m.viewport.Width = msg.Width m.viewport.Height = max(msg.Height-3, 1) + m.viewport.SetContent(m.wrappedDetail()) m.offset = m.clampedOffset() return m.maybeFetch() @@ -217,7 +219,7 @@ func (m listModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.detailContent = msg.body } m.detailTitle = msg.title - m.viewport.SetContent(m.detailContent) + m.viewport.SetContent(m.wrappedDetail()) m.viewport.GotoTop() return m, nil @@ -292,6 +294,13 @@ func (m listModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m, nil } +// wrappedDetail is the detail text wrapped to the pane width: the viewport clips +// lines at the pane edge and can't scroll horizontally, which would hide the end +// of a long failure reason or log line. +func (m listModel) wrappedDetail() string { + return ansi.Wrap(m.detailContent, m.viewport.Width, "") +} + // clampedOffset returns the scroll offset that keeps the cursor visible. func (m listModel) clampedOffset() int { visible := m.visibleCount() diff --git a/cmd/air/list_tui_test.go b/cmd/air/list_tui_test.go index 583cc570c07..c52791f23a5 100644 --- a/cmd/air/list_tui_test.go +++ b/cmd/air/list_tui_test.go @@ -3,9 +3,11 @@ package aircmd import ( "io" "strconv" + "strings" "testing" tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/x/ansi" "github.com/databricks/cli/libs/cmdio" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -269,6 +271,25 @@ func TestListModelDetailPaneAndBack(t *testing.T) { assert.Contains(t, m.View(), "Run ID") } +func TestListModelDetailPaneWrapsLongLines(t *testing.T) { + m := detailLoadingModel(t) + reason := "Failure reason: " + strings.Repeat("the task exited with a long error ", 4) + "ENDOFREASON" + require.Greater(t, len(reason), 80) + + next, _ := m.Update(detailMsg{title: "Run Details", body: reason}) + m = next.(listModel) + assert.Contains(t, m.View(), "ENDOFREASON", "the end of a long line must be visible, not clipped at the pane edge") + + // A narrower terminal re-wraps the same text. + next, _ = m.Update(tea.WindowSizeMsg{Width: 30, Height: 24}) + m = next.(listModel) + view := m.View() + assert.Contains(t, view, "ENDOFREASON") + for line := range strings.SplitSeq(view, "\n") { + assert.LessOrEqual(t, ansi.StringWidth(line), 30, line) + } +} + func TestListModelDetailLateMsgDropped(t *testing.T) { m := detailLoadingModel(t) diff --git a/cmd/air/logstream_support.go b/cmd/air/logstream_support.go index 69896f71c3c..a099a286a01 100644 --- a/cmd/air/logstream_support.go +++ b/cmd/air/logstream_support.go @@ -234,6 +234,11 @@ func emitNoLogs(out io.Writer, req logRequest, status logRunStatus) { if status.stateMessage != "" { msg = fmt.Sprintf("%s: %s", msg, status.stateMessage) } + // A retry that failed before it started has no logs of its own, but the + // attempts before it may. + if req.attempt < 0 && status.terminal() && status.latestAttempt > 0 { + msg = fmt.Sprintf("%s. Earlier retries may have logs: databricks air logs %d --retry %d", strings.TrimRight(msg, "."), req.runID, status.latestAttempt-1) + } if req.jsonOutput { printLogEvent(out, "ERROR", req.node, msg) return diff --git a/cmd/air/logstream_test.go b/cmd/air/logstream_test.go index fe0756d3862..98bd92d0d3d 100644 --- a/cmd/air/logstream_test.go +++ b/cmd/air/logstream_test.go @@ -530,6 +530,7 @@ func TestMatchFatalPattern(t *testing.T) { func TestEmitNoLogs(t *testing.T) { tests := []struct { name string + req logRequest status logRunStatus want string }{ @@ -543,15 +544,30 @@ func TestEmitNoLogs(t *testing.T) { status: logRunStatus{lifeCycleState: "RUNNING"}, want: "No logs available yet for run 7, which is still in state RUNNING\n", }, + { + name: "latest retry of a terminal run", + req: logRequest{attempt: -1}, + status: logRunStatus{lifeCycleState: "TERMINATED", resultState: "FAILED", stateMessage: "Out of capacity.", latestAttempt: 2}, + want: "No logs available for run 7. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs: databricks air logs 7 --retry 1\n", + }, + { + name: "explicitly requested retry", + req: logRequest{attempt: 2}, + status: logRunStatus{lifeCycleState: "TERMINATED", resultState: "FAILED", latestAttempt: 2}, + want: "No logs available for run 7. Run terminated in state FAILED\n", + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var text bytes.Buffer - emitNoLogs(&text, logRequest{runID: 7}, tc.status) + req := tc.req + req.runID = 7 + emitNoLogs(&text, req, tc.status) assert.Equal(t, tc.want, text.String()) var jsonBuf bytes.Buffer - emitNoLogs(&jsonBuf, logRequest{runID: 7, node: 1, jsonOutput: true}, tc.status) + req.node, req.jsonOutput = 1, true + emitNoLogs(&jsonBuf, req, tc.status) var ev logEvent require.NoError(t, json.Unmarshal(jsonBuf.Bytes(), &ev)) assert.Equal(t, "ERROR", ev.Type) diff --git a/cmd/air/mlflow.go b/cmd/air/mlflow.go index 5434142a3c8..5ce874e7a05 100644 --- a/cmd/air/mlflow.go +++ b/cmd/air/mlflow.go @@ -3,6 +3,7 @@ package aircmd import ( "context" "fmt" + "slices" "strings" "github.com/databricks/cli/libs/log" @@ -17,10 +18,27 @@ type mlflowIdentifiers struct { RunID string } -// mlflowIDs fetches the MLflow IDs for a run via its latest task. Returns nil if -// they can't be obtained. +// mlflowIDs fetches the MLflow IDs for a run via its latest task, falling back +// to earlier attempts (see mlflowIDsFromAttempts). Returns nil if they can't be +// obtained. func mlflowIDs(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run) *mlflowIdentifiers { - return mlflowIDsFromOutput(aiRuntimeTaskOutput(ctx, w, run)) + return mlflowIDsFromAttempts(ctx, w, run, aiRuntimeTaskOutput(ctx, w, run)) +} + +// mlflowIDsFromAttempts returns the MLflow IDs in latest, the latest attempt's +// output, else those of the newest earlier attempt that has them. A retry that +// failed before it started has no MLflow output, but the run's earlier +// attempts still logged to an MLflow run worth linking. +func mlflowIDsFromAttempts(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run, latest *jobs.AiRuntimeTaskOutput) *mlflowIdentifiers { + if ids := mlflowIDsFromOutput(latest); ids != nil || len(run.Tasks) < 2 { + return ids + } + for _, task := range slices.Backward(run.Tasks[:len(run.Tasks)-1]) { + if ids := mlflowIDsForTask(ctx, w, task.RunId); ids != nil { + return ids + } + } + return nil } func aiRuntimeTaskOutput(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run) *jobs.AiRuntimeTaskOutput { diff --git a/cmd/air/mlflow_test.go b/cmd/air/mlflow_test.go index a3dd5376298..8692e0dfc15 100644 --- a/cmd/air/mlflow_test.go +++ b/cmd/air/mlflow_test.go @@ -63,18 +63,35 @@ func TestMLflowIDs(t *testing.T) { t.Run("uses the latest attempt's task run", func(t *testing.T) { // A retried run must link to the last task, not the stale first attempt. - var gotRunID string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/api/2.2/jobs/runs/get-output" { - gotRunID = r.URL.Query().Get("run_id") + runID := r.URL.Query().Get("run_id") + _, _ = w.Write([]byte(`{"ai_runtime_task_output":{"mlflow_experiment_id":"E1","mlflow_run_id":"R` + runID + `"}}`)) + return + } + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + + retried := &jobs.Run{Tasks: []jobs.RunTask{{RunId: 99}, {RunId: 100}}} + got := mlflowIDs(ctx, newTestWorkspaceClient(t, srv.URL), retried) + assert.Equal(t, &mlflowIdentifiers{ExperimentID: "E1", RunID: "R100"}, got) + }) + + t.Run("falls back to an earlier attempt when the latest has none", func(t *testing.T) { + // The last retry failed before it started, so only attempt 0 logged to MLflow. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/2.2/jobs/runs/get-output" && r.URL.Query().Get("run_id") == "99" { + _, _ = w.Write([]byte(`{"ai_runtime_task_output":{"mlflow_experiment_id":"E1","mlflow_run_id":"R1"}}`)) + return } _, _ = w.Write([]byte(`{}`)) })) t.Cleanup(srv.Close) retried := &jobs.Run{Tasks: []jobs.RunTask{{RunId: 99}, {RunId: 100}}} - mlflowIDs(ctx, newTestWorkspaceClient(t, srv.URL), retried) - assert.Equal(t, "100", gotRunID) + got := mlflowIDs(ctx, newTestWorkspaceClient(t, srv.URL), retried) + assert.Equal(t, &mlflowIdentifiers{ExperimentID: "E1", RunID: "R1"}, got) }) } From a52b44580570a992502a3c239c981c081fe50b98 Mon Sep 17 00:00:00 2001 From: Pardis Beikzadeh Date: Tue, 10 Nov 2026 23:40:53 +0000 Subject: [PATCH 02/10] air: Keep box rows and URLs unwrapped; fall back to earlier MLflow IDs only for finished runs Co-authored-by: Isaac --- acceptance/air/logs-failed-retry/output.txt | 2 +- cmd/air/list_detail_test.go | 2 +- cmd/air/list_tui.go | 27 ++++++++++++++++----- cmd/air/list_tui_test.go | 23 +++++++++++++++--- cmd/air/logstream_support.go | 5 ++-- cmd/air/logstream_test.go | 8 +++++- cmd/air/mlflow.go | 11 +++++---- cmd/air/mlflow_test.go | 13 +++++++--- 8 files changed, 67 insertions(+), 24 deletions(-) diff --git a/acceptance/air/logs-failed-retry/output.txt b/acceptance/air/logs-failed-retry/output.txt index 7bdcc1c0562..41e3669ad8d 100644 --- a/acceptance/air/logs-failed-retry/output.txt +++ b/acceptance/air/logs-failed-retry/output.txt @@ -1,6 +1,6 @@ === logs of a failed retry with no logs point to the earlier retry >>> [CLI] air logs 123 -No logs available for run 123. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs: databricks air logs 123 --retry 0 +No logs available for run 123. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs; pass --retry 0 to see them Exit code: 1 diff --git a/cmd/air/list_detail_test.go b/cmd/air/list_detail_test.go index b3d2cde9df5..7c88b347883 100644 --- a/cmd/air/list_detail_test.go +++ b/cmd/air/list_detail_test.go @@ -32,6 +32,6 @@ func TestRunLogsSnapshotFailedRetryWithoutLogs(t *testing.T) { got, err := runLogsSnapshot(t.Context(), newTestWorkspaceClient(t, srv.URL), 123) require.NoError(t, err) assert.Equal(t, "No logs available for run 123. Run terminated in state FAILED: Out of capacity. "+ - "Earlier retries may have logs: databricks air logs 123 --retry 0\n"+ + "Earlier retries may have logs; pass --retry 0 to see them\n"+ "\nMLflow URL: "+srv.URL+"/ml/experiments/E1/runs/R1/artifacts/logs/node_0\n", got) } diff --git a/cmd/air/list_tui.go b/cmd/air/list_tui.go index ea2264e8ad0..af5951bca44 100644 --- a/cmd/air/list_tui.go +++ b/cmd/air/list_tui.go @@ -123,7 +123,12 @@ type listModel struct { detailContent string // rendered detail content } +// detailHorizontalStep is how many columns ←/→ pan the detail pane. +const detailHorizontalStep = 8 + func newListModel(r *lipgloss.Renderer, f *runFetcher, rows []listRow, links bool) listModel { + vp := viewport.New(0, 0) + vp.SetHorizontalStep(detailHorizontalStep) return listModel{ rows: rows, styles: newListStyles(r), @@ -131,7 +136,7 @@ func newListModel(r *lipgloss.Renderer, f *runFetcher, rows []listRow, links boo links: links, fetcher: f, mode: modeList, - viewport: viewport.New(0, 0), + viewport: vp, } } @@ -221,6 +226,7 @@ func (m listModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.detailTitle = msg.title m.viewport.SetContent(m.wrappedDetail()) m.viewport.GotoTop() + m.viewport.SetXOffset(0) return m, nil case tea.KeyMsg: @@ -294,11 +300,20 @@ func (m listModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m, nil } -// wrappedDetail is the detail text wrapped to the pane width: the viewport clips -// lines at the pane edge and can't scroll horizontally, which would hide the end -// of a long failure reason or log line. +// wrappedDetail is the detail text with long lines wrapped to the pane width, +// since the viewport clips them at the pane edge. Box rows and URLs stay on one +// row, because wrapping would break the box borders or the link; ←/→ pans to +// their clipped ends. func (m listModel) wrappedDetail() string { - return ansi.Wrap(m.detailContent, m.viewport.Width, "") + lines := strings.Split(m.detailContent, "\n") + for i, line := range lines { + plain := strings.TrimSpace(ansi.Strip(line)) + if strings.IndexAny(plain, "╭│╰") == 0 || strings.Contains(plain, "://") { + continue + } + lines[i] = ansi.Wrap(line, m.viewport.Width, "") + } + return strings.Join(lines, "\n") } // clampedOffset returns the scroll offset that keeps the cursor visible. @@ -320,7 +335,7 @@ func (m listModel) View() string { } else { lines = append(lines, m.viewport.View()) } - footer := faint.Render("↑/↓ scroll · esc back · q quit") + footer := faint.Render("↑/↓ scroll · ←/→ pan · esc back · q quit") lines = append(lines, footer) return strings.Join(lines, "\n") } diff --git a/cmd/air/list_tui_test.go b/cmd/air/list_tui_test.go index c52791f23a5..fafb04588da 100644 --- a/cmd/air/list_tui_test.go +++ b/cmd/air/list_tui_test.go @@ -283,13 +283,30 @@ func TestListModelDetailPaneWrapsLongLines(t *testing.T) { // A narrower terminal re-wraps the same text. next, _ = m.Update(tea.WindowSizeMsg{Width: 30, Height: 24}) m = next.(listModel) - view := m.View() - assert.Contains(t, view, "ENDOFREASON") - for line := range strings.SplitSeq(view, "\n") { + pane := m.viewport.View() + assert.Contains(t, pane, "ENDOFREASON") + for line := range strings.SplitSeq(pane, "\n") { assert.LessOrEqual(t, ansi.StringWidth(line), 30, line) } } +func TestListModelDetailPaneKeepsBoxRowsAndURLsWhole(t *testing.T) { + m := detailLoadingModel(t) + boxRow := "│ command: python train.py " + strings.Repeat("--flag ", 12) + "│" + url := "MLflow URL: https://example.test/ml/experiments/1/runs/" + strings.Repeat("a", 64) + require.Greater(t, len(boxRow), 80) + require.Greater(t, len(url), 80) + + next, _ := m.Update(detailMsg{title: "Run Details", body: boxRow + "\n" + url}) + m = next.(listModel) + assert.Equal(t, 2, m.viewport.TotalLineCount(), "box rows and URLs must not wrap") + + // ←/→ pans to the clipped end instead. + next, _ = m.Update(tea.KeyMsg{Type: tea.KeyRight}) + m = next.(listModel) + assert.Positive(t, m.viewport.HorizontalScrollPercent()) +} + func TestListModelDetailLateMsgDropped(t *testing.T) { m := detailLoadingModel(t) diff --git a/cmd/air/logstream_support.go b/cmd/air/logstream_support.go index a099a286a01..41a5f83c5d8 100644 --- a/cmd/air/logstream_support.go +++ b/cmd/air/logstream_support.go @@ -234,10 +234,9 @@ func emitNoLogs(out io.Writer, req logRequest, status logRunStatus) { if status.stateMessage != "" { msg = fmt.Sprintf("%s: %s", msg, status.stateMessage) } - // A retry that failed before it started has no logs of its own, but the - // attempts before it may. + // Only the latest attempt was checked; earlier ones may have logs. if req.attempt < 0 && status.terminal() && status.latestAttempt > 0 { - msg = fmt.Sprintf("%s. Earlier retries may have logs: databricks air logs %d --retry %d", strings.TrimRight(msg, "."), req.runID, status.latestAttempt-1) + msg = fmt.Sprintf("%s. Earlier retries may have logs; pass --retry %d to see them", strings.TrimRight(msg, "."), status.latestAttempt-1) } if req.jsonOutput { printLogEvent(out, "ERROR", req.node, msg) diff --git a/cmd/air/logstream_test.go b/cmd/air/logstream_test.go index 98bd92d0d3d..4afdf2e412f 100644 --- a/cmd/air/logstream_test.go +++ b/cmd/air/logstream_test.go @@ -548,7 +548,13 @@ func TestEmitNoLogs(t *testing.T) { name: "latest retry of a terminal run", req: logRequest{attempt: -1}, status: logRunStatus{lifeCycleState: "TERMINATED", resultState: "FAILED", stateMessage: "Out of capacity.", latestAttempt: 2}, - want: "No logs available for run 7. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs: databricks air logs 7 --retry 1\n", + want: "No logs available for run 7. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs; pass --retry 1 to see them\n", + }, + { + name: "latest retry of a running run", + req: logRequest{attempt: -1}, + status: logRunStatus{lifeCycleState: "RUNNING", latestAttempt: 2}, + want: "No logs available yet for run 7, which is still in state RUNNING\n", }, { name: "explicitly requested retry", diff --git a/cmd/air/mlflow.go b/cmd/air/mlflow.go index 5ce874e7a05..4149556580d 100644 --- a/cmd/air/mlflow.go +++ b/cmd/air/mlflow.go @@ -25,12 +25,13 @@ func mlflowIDs(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run return mlflowIDsFromAttempts(ctx, w, run, aiRuntimeTaskOutput(ctx, w, run)) } -// mlflowIDsFromAttempts returns the MLflow IDs in latest, the latest attempt's -// output, else those of the newest earlier attempt that has them. A retry that -// failed before it started has no MLflow output, but the run's earlier -// attempts still logged to an MLflow run worth linking. +// mlflowIDsFromAttempts returns the IDs from latest (the latest attempt's +// output), else, for a finished run, from the newest earlier attempt that has +// them: a last retry that failed before it started never got an MLflow run of +// its own. A run still in progress keeps only the latest attempt's IDs, so its +// link and ETA never point at a previous attempt. func mlflowIDsFromAttempts(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run, latest *jobs.AiRuntimeTaskOutput) *mlflowIdentifiers { - if ids := mlflowIDsFromOutput(latest); ids != nil || len(run.Tasks) < 2 { + if ids := mlflowIDsFromOutput(latest); ids != nil || len(run.Tasks) < 2 || !isTerminal(run) { return ids } for _, task := range slices.Backward(run.Tasks[:len(run.Tasks)-1]) { diff --git a/cmd/air/mlflow_test.go b/cmd/air/mlflow_test.go index 8692e0dfc15..bde83ad87a5 100644 --- a/cmd/air/mlflow_test.go +++ b/cmd/air/mlflow_test.go @@ -78,7 +78,7 @@ func TestMLflowIDs(t *testing.T) { assert.Equal(t, &mlflowIdentifiers{ExperimentID: "E1", RunID: "R100"}, got) }) - t.Run("falls back to an earlier attempt when the latest has none", func(t *testing.T) { + t.Run("a finished run falls back to an earlier attempt when the latest has none", func(t *testing.T) { // The last retry failed before it started, so only attempt 0 logged to MLflow. srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/api/2.2/jobs/runs/get-output" && r.URL.Query().Get("run_id") == "99" { @@ -89,9 +89,14 @@ func TestMLflowIDs(t *testing.T) { })) t.Cleanup(srv.Close) - retried := &jobs.Run{Tasks: []jobs.RunTask{{RunId: 99}, {RunId: 100}}} - got := mlflowIDs(ctx, newTestWorkspaceClient(t, srv.URL), retried) - assert.Equal(t, &mlflowIdentifiers{ExperimentID: "E1", RunID: "R1"}, got) + w := newTestWorkspaceClient(t, srv.URL) + tasks := []jobs.RunTask{{RunId: 99}, {RunId: 100}} + finished := &jobs.Run{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated}, Tasks: tasks} + assert.Equal(t, &mlflowIdentifiers{ExperimentID: "E1", RunID: "R1"}, mlflowIDs(ctx, w, finished)) + + // A run still starting its next retry must not link to the previous attempt. + running := &jobs.Run{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, Tasks: tasks} + assert.Nil(t, mlflowIDs(ctx, w, running)) }) } From b5fef6d1531b72b928c352610b26741edef678ee Mon Sep 17 00:00:00 2001 From: Pardis Beikzadeh Date: Tue, 10 Nov 2026 23:41:47 +0000 Subject: [PATCH 03/10] Link changelog entry to this PR Co-authored-by: Isaac --- .nextchanges/air/{air-list-detail-panes.md => 7456.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .nextchanges/air/{air-list-detail-panes.md => 7456.md} (100%) diff --git a/.nextchanges/air/air-list-detail-panes.md b/.nextchanges/air/7456.md similarity index 100% rename from .nextchanges/air/air-list-detail-panes.md rename to .nextchanges/air/7456.md From ced2db05803ab3fe02d6c6a7b3890eedf2378de3 Mon Sep 17 00:00:00 2001 From: Pardis Beikzadeh Date: Tue, 10 Nov 2026 23:45:37 +0000 Subject: [PATCH 04/10] Add PR link to changelog entry Co-authored-by: Isaac --- .nextchanges/air/7456.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/air/7456.md b/.nextchanges/air/7456.md index 26f1e2fb4fd..d8aeec7ff6b 100644 --- a/.nextchanges/air/7456.md +++ b/.nextchanges/air/7456.md @@ -1 +1 @@ -* Wrap long failure reasons and log lines in the `air list` info and logs panes, keep the MLflow link when a run's last retry failed before starting, and point `air logs` at earlier retries when the last one has no logs. +* Wrap long failure reasons and log lines in the `air list` info and logs panes, keep the MLflow link when a run's last retry failed before starting, and point `air logs` at earlier retries when the last one has no logs. ([#7456](https://github.com/databricks/cli/pull/7456)) From 7d8321b5efcc6661e070ab40e4b1778a525aba37 Mon Sep 17 00:00:00 2001 From: Pardis Beikzadeh Date: Wed, 11 Nov 2026 16:39:18 +0000 Subject: [PATCH 05/10] air: Give the retry range in the no-logs hint; fall back only to retries of the same task The hint named the newest earlier retry, which can be empty too. The MLflow fallback also linked other tasks' runs in multi-task jobs. Co-authored-by: Isaac --- acceptance/air/logs-failed-retry/output.txt | 4 ++-- acceptance/air/logs-failed-retry/script | 2 +- acceptance/air/logs-failed-retry/test.toml | 5 +++-- cmd/air/list_detail_test.go | 10 +++++----- cmd/air/logstream_support.go | 6 +++++- cmd/air/logstream_test.go | 8 +++++++- cmd/air/mlflow.go | 12 ++++++++---- cmd/air/mlflow_test.go | 7 +++++++ 8 files changed, 38 insertions(+), 16 deletions(-) diff --git a/acceptance/air/logs-failed-retry/output.txt b/acceptance/air/logs-failed-retry/output.txt index 41e3669ad8d..3d468dec762 100644 --- a/acceptance/air/logs-failed-retry/output.txt +++ b/acceptance/air/logs-failed-retry/output.txt @@ -1,6 +1,6 @@ -=== logs of a failed retry with no logs point to the earlier retry +=== logs of a failed retry with no logs point to earlier retries >>> [CLI] air logs 123 -No logs available for run 123. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs; pass --retry 0 to see them +No logs available for run 123. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs; pass --retry N, from 0 to 1, to see them Exit code: 1 diff --git a/acceptance/air/logs-failed-retry/script b/acceptance/air/logs-failed-retry/script index 741e68c28b2..f76fb66a16f 100644 --- a/acceptance/air/logs-failed-retry/script +++ b/acceptance/air/logs-failed-retry/script @@ -1,2 +1,2 @@ -title "logs of a failed retry with no logs point to the earlier retry" +title "logs of a failed retry with no logs point to earlier retries" errcode trace $CLI air logs 123 diff --git a/acceptance/air/logs-failed-retry/test.toml b/acceptance/air/logs-failed-retry/test.toml index 14224ee8f14..d9f7cf4ae81 100644 --- a/acceptance/air/logs-failed-retry/test.toml +++ b/acceptance/air/logs-failed-retry/test.toml @@ -4,7 +4,7 @@ Pattern = "HEAD /" Response.Body = '' -# A failed run whose second attempt failed before it started. +# A failed run whose second and third attempts failed before they started. [[Server]] Pattern = "GET /api/2.2/jobs/runs/get" Response.Body = ''' @@ -15,7 +15,8 @@ Response.Body = ''' "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "Out of capacity"}, "tasks": [ {"task_key": "train", "run_id": 456, "attempt_number": 0}, - {"task_key": "train", "run_id": 457, "attempt_number": 1} + {"task_key": "train", "run_id": 457, "attempt_number": 1}, + {"task_key": "train", "run_id": 458, "attempt_number": 2} ] } ''' diff --git a/cmd/air/list_detail_test.go b/cmd/air/list_detail_test.go index 7c88b347883..a1678e130fe 100644 --- a/cmd/air/list_detail_test.go +++ b/cmd/air/list_detail_test.go @@ -11,16 +11,16 @@ import ( ) func TestRunLogsSnapshotFailedRetryWithoutLogs(t *testing.T) { - // The last retry failed before it started: no logs and no MLflow output of - // its own. The pane explains that and still links the MLflow run that - // attempt 0 logged to. + // Only attempt 0 ran: retries 1 and 2 failed before they started, with no + // logs or MLflow output of their own. The pane still links the MLflow run + // that attempt 0 logged to. srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case strings.HasSuffix(r.URL.Path, "/logs"): _, _ = w.Write([]byte(`{"log_records": []}`)) case r.URL.Path == "/api/2.2/jobs/runs/get": _, _ = w.Write([]byte(`{"run_id": 123, "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "Out of capacity"}, - "tasks": [{"run_id": 456, "attempt_number": 0}, {"run_id": 457, "attempt_number": 1}]}`)) + "tasks": [{"run_id": 456, "attempt_number": 0}, {"run_id": 457, "attempt_number": 1}, {"run_id": 458, "attempt_number": 2}]}`)) case r.URL.Path == "/api/2.2/jobs/runs/get-output" && r.URL.Query().Get("run_id") == "456": _, _ = w.Write([]byte(`{"ai_runtime_task_output": {"mlflow_experiment_id": "E1", "mlflow_run_id": "R1"}}`)) default: @@ -32,6 +32,6 @@ func TestRunLogsSnapshotFailedRetryWithoutLogs(t *testing.T) { got, err := runLogsSnapshot(t.Context(), newTestWorkspaceClient(t, srv.URL), 123) require.NoError(t, err) assert.Equal(t, "No logs available for run 123. Run terminated in state FAILED: Out of capacity. "+ - "Earlier retries may have logs; pass --retry 0 to see them\n"+ + "Earlier retries may have logs; pass --retry N, from 0 to 1, to see them\n"+ "\nMLflow URL: "+srv.URL+"/ml/experiments/E1/runs/R1/artifacts/logs/node_0\n", got) } diff --git a/cmd/air/logstream_support.go b/cmd/air/logstream_support.go index 41a5f83c5d8..9817445decc 100644 --- a/cmd/air/logstream_support.go +++ b/cmd/air/logstream_support.go @@ -236,7 +236,11 @@ func emitNoLogs(out io.Writer, req logRequest, status logRunStatus) { } // Only the latest attempt was checked; earlier ones may have logs. if req.attempt < 0 && status.terminal() && status.latestAttempt > 0 { - msg = fmt.Sprintf("%s. Earlier retries may have logs; pass --retry %d to see them", strings.TrimRight(msg, "."), status.latestAttempt-1) + retry := "0" + if status.latestAttempt > 1 { + retry = fmt.Sprintf("N, from 0 to %d,", status.latestAttempt-1) + } + msg = fmt.Sprintf("%s. Earlier retries may have logs; pass --retry %s to see them", strings.TrimRight(msg, "."), retry) } if req.jsonOutput { printLogEvent(out, "ERROR", req.node, msg) diff --git a/cmd/air/logstream_test.go b/cmd/air/logstream_test.go index 4afdf2e412f..1d76c07d9fb 100644 --- a/cmd/air/logstream_test.go +++ b/cmd/air/logstream_test.go @@ -548,7 +548,13 @@ func TestEmitNoLogs(t *testing.T) { name: "latest retry of a terminal run", req: logRequest{attempt: -1}, status: logRunStatus{lifeCycleState: "TERMINATED", resultState: "FAILED", stateMessage: "Out of capacity.", latestAttempt: 2}, - want: "No logs available for run 7. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs; pass --retry 1 to see them\n", + want: "No logs available for run 7. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs; pass --retry N, from 0 to 1, to see them\n", + }, + { + name: "latest retry of a terminal run with one earlier retry", + req: logRequest{attempt: -1}, + status: logRunStatus{lifeCycleState: "TERMINATED", resultState: "FAILED", latestAttempt: 1}, + want: "No logs available for run 7. Run terminated in state FAILED. Earlier retries may have logs; pass --retry 0 to see them\n", }, { name: "latest retry of a running run", diff --git a/cmd/air/mlflow.go b/cmd/air/mlflow.go index 4149556580d..b5fae13a493 100644 --- a/cmd/air/mlflow.go +++ b/cmd/air/mlflow.go @@ -26,15 +26,19 @@ func mlflowIDs(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run } // mlflowIDsFromAttempts returns the IDs from latest (the latest attempt's -// output), else, for a finished run, from the newest earlier attempt that has -// them: a last retry that failed before it started never got an MLflow run of -// its own. A run still in progress keeps only the latest attempt's IDs, so its -// link and ETA never point at a previous attempt. +// output), else, for a finished run, from the newest earlier retry of the same +// task that has them: a last retry that failed before it started never got an +// MLflow run of its own. A run still in progress keeps only the latest attempt's +// IDs, so its link and ETA never point at a previous attempt. func mlflowIDsFromAttempts(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run, latest *jobs.AiRuntimeTaskOutput) *mlflowIdentifiers { if ids := mlflowIDsFromOutput(latest); ids != nil || len(run.Tasks) < 2 || !isTerminal(run) { return ids } + last := run.Tasks[len(run.Tasks)-1] for _, task := range slices.Backward(run.Tasks[:len(run.Tasks)-1]) { + if task.TaskKey != last.TaskKey { + continue + } if ids := mlflowIDsForTask(ctx, w, task.RunId); ids != nil { return ids } diff --git a/cmd/air/mlflow_test.go b/cmd/air/mlflow_test.go index bde83ad87a5..10ab8e6c477 100644 --- a/cmd/air/mlflow_test.go +++ b/cmd/air/mlflow_test.go @@ -97,6 +97,13 @@ func TestMLflowIDs(t *testing.T) { // A run still starting its next retry must not link to the previous attempt. running := &jobs.Run{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, Tasks: tasks} assert.Nil(t, mlflowIDs(ctx, w, running)) + + // In a multi-task job, an earlier task is not a retry of the last one. + multiTask := &jobs.Run{ + State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated}, + Tasks: []jobs.RunTask{{TaskKey: "train", RunId: 99}, {TaskKey: "report", RunId: 100}}, + } + assert.Nil(t, mlflowIDs(ctx, w, multiTask)) }) } From b342a39d86a27d07c088a87f57366d7c2d9fe2af Mon Sep 17 00:00:00 2001 From: Pardis Beikzadeh Date: Wed, 11 Nov 2026 17:10:00 +0000 Subject: [PATCH 06/10] Update logs-failed-retry output for the title newline from main Co-authored-by: Isaac --- acceptance/air/logs-failed-retry/output.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/acceptance/air/logs-failed-retry/output.txt b/acceptance/air/logs-failed-retry/output.txt index 3d468dec762..a135296c266 100644 --- a/acceptance/air/logs-failed-retry/output.txt +++ b/acceptance/air/logs-failed-retry/output.txt @@ -1,5 +1,6 @@ === logs of a failed retry with no logs point to earlier retries + >>> [CLI] air logs 123 No logs available for run 123. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs; pass --retry N, from 0 to 1, to see them From 8d75f65572981baaf526327e30c4c2901a8dfa88 Mon Sep 17 00:00:00 2001 From: Pardis Beikzadeh Date: Thu, 12 Nov 2026 00:04:13 +0000 Subject: [PATCH 07/10] air: Leave the MLflow link empty when a retry's output can't be fetched Falling back past a retry whose get-output failed could link an older MLflow run while the newest one is unknown. Co-authored-by: Isaac --- cmd/air/get.go | 4 +-- cmd/air/mlflow.go | 65 ++++++++++++++++++++++++++++-------------- cmd/air/mlflow_test.go | 32 ++++++++++++++++++++- 3 files changed, 76 insertions(+), 25 deletions(-) diff --git a/cmd/air/get.go b/cmd/air/get.go index 6466aab7a20..09886e6b6f6 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -164,8 +164,8 @@ JOB_RUN_ID is the job run ID returned by "air run".`, data := buildGetData(run) data.DashboardURL = dashboardURL(w.Config.Host, runID, workspaceID) - taskOutput := aiRuntimeTaskOutput(ctx, w, run) - ids := mlflowIDsFromAttempts(ctx, w, run, taskOutput) + taskOutput, taskOutputErr := latestTaskOutput(ctx, w, run.Tasks) + ids := mlflowIDsFromRetries(ctx, w, run.Tasks, isTerminal(run), taskOutput, taskOutputErr) if ids != nil { url := mlflowLogsURL(w.Config.Host, ids) data.MLflowURL = &url diff --git a/cmd/air/mlflow.go b/cmd/air/mlflow.go index b5fae13a493..961b8d82099 100644 --- a/cmd/air/mlflow.go +++ b/cmd/air/mlflow.go @@ -18,28 +18,39 @@ type mlflowIdentifiers struct { RunID string } -// mlflowIDs fetches the MLflow IDs for a run via its latest task, falling back -// to earlier attempts (see mlflowIDsFromAttempts). Returns nil if they can't be -// obtained. +// mlflowIDs fetches the MLflow IDs a run's views link to (see +// mlflowIDsFromRetries). Returns nil if they can't be obtained. func mlflowIDs(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run) *mlflowIdentifiers { - return mlflowIDsFromAttempts(ctx, w, run, aiRuntimeTaskOutput(ctx, w, run)) + latest, err := latestTaskOutput(ctx, w, run.Tasks) + return mlflowIDsFromRetries(ctx, w, run.Tasks, isTerminal(run), latest, err) } -// mlflowIDsFromAttempts returns the IDs from latest (the latest attempt's -// output), else, for a finished run, from the newest earlier retry of the same -// task that has them: a last retry that failed before it started never got an -// MLflow run of its own. A run still in progress keeps only the latest attempt's -// IDs, so its link and ETA never point at a previous attempt. -func mlflowIDsFromAttempts(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run, latest *jobs.AiRuntimeTaskOutput) *mlflowIdentifiers { - if ids := mlflowIDsFromOutput(latest); ids != nil || len(run.Tasks) < 2 || !isTerminal(run) { +// mlflowIDsFromRetries returns the IDs from latest (the latest retry's output), +// else, for a finished run, from the newest earlier retry of the same task that +// has them: a last retry that failed before it started never got an MLflow run +// of its own. A run still in progress keeps only the latest retry's IDs, so its +// link and ETA never point at a previous retry. +// +// If a get-output call fails (latestErr, or an earlier retry's), it returns nil +// rather than skip that retry: the retry it couldn't read may have the newest +// MLflow run, and an older link would then look current. +func mlflowIDsFromRetries(ctx context.Context, w *databricks.WorkspaceClient, tasks []jobs.RunTask, terminal bool, latest *jobs.AiRuntimeTaskOutput, latestErr error) *mlflowIdentifiers { + if latestErr != nil { + return nil + } + if ids := mlflowIDsFromOutput(latest); ids != nil || len(tasks) < 2 || !terminal { return ids } - last := run.Tasks[len(run.Tasks)-1] - for _, task := range slices.Backward(run.Tasks[:len(run.Tasks)-1]) { + last := tasks[len(tasks)-1] + for _, task := range slices.Backward(tasks[:len(tasks)-1]) { if task.TaskKey != last.TaskKey { continue } - if ids := mlflowIDsForTask(ctx, w, task.RunId); ids != nil { + out, err := aiRuntimeTaskOutputForTask(ctx, w, task.RunId) + if err != nil { + return nil + } + if ids := mlflowIDsFromOutput(out); ids != nil { return ids } } @@ -47,10 +58,17 @@ func mlflowIDsFromAttempts(ctx context.Context, w *databricks.WorkspaceClient, r } func aiRuntimeTaskOutput(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run) *jobs.AiRuntimeTaskOutput { - if len(run.Tasks) == 0 { - return nil + out, _ := latestTaskOutput(ctx, w, run.Tasks) + return out +} + +// latestTaskOutput fetches the output of the last of tasks, the run's latest +// retry (see aiRuntimeTaskOutputForTask). +func latestTaskOutput(ctx context.Context, w *databricks.WorkspaceClient, tasks []jobs.RunTask) (*jobs.AiRuntimeTaskOutput, error) { + if len(tasks) == 0 { + return nil, nil } - return aiRuntimeTaskOutputForTask(ctx, w, run.Tasks[len(run.Tasks)-1].RunId) + return aiRuntimeTaskOutputForTask(ctx, w, tasks[len(tasks)-1].RunId) } // mlflowIDsForTask fetches a task run's MLflow experiment and run IDs from @@ -58,21 +76,24 @@ func aiRuntimeTaskOutput(ctx context.Context, w *databricks.WorkspaceClient, run // link, so any failure (endpoint error, run not yet started, no MLflow output) // is logged and treated as "no link" rather than failing the command. func mlflowIDsForTask(ctx context.Context, w *databricks.WorkspaceClient, taskRunID int64) *mlflowIdentifiers { - return mlflowIDsFromOutput(aiRuntimeTaskOutputForTask(ctx, w, taskRunID)) + out, _ := aiRuntimeTaskOutputForTask(ctx, w, taskRunID) + return mlflowIDsFromOutput(out) } -func aiRuntimeTaskOutputForTask(ctx context.Context, w *databricks.WorkspaceClient, taskRunID int64) *jobs.AiRuntimeTaskOutput { +// aiRuntimeTaskOutputForTask fetches a task run's AI runtime output, logging +// any fetch error before returning it. +func aiRuntimeTaskOutputForTask(ctx context.Context, w *databricks.WorkspaceClient, taskRunID int64) (*jobs.AiRuntimeTaskOutput, error) { if taskRunID == 0 { - return nil + return nil, nil } out, err := w.Jobs.GetRunOutputByRunId(ctx, taskRunID) if err != nil { log.Debugf(ctx, "air: could not fetch run output for MLflow link: %v", err) - return nil + return nil, err } - return out.AiRuntimeTaskOutput + return out.AiRuntimeTaskOutput, nil } func mlflowIDsFromOutput(output *jobs.AiRuntimeTaskOutput) *mlflowIdentifiers { diff --git a/cmd/air/mlflow_test.go b/cmd/air/mlflow_test.go index 10ab8e6c477..b08fa971465 100644 --- a/cmd/air/mlflow_test.go +++ b/cmd/air/mlflow_test.go @@ -90,7 +90,7 @@ func TestMLflowIDs(t *testing.T) { t.Cleanup(srv.Close) w := newTestWorkspaceClient(t, srv.URL) - tasks := []jobs.RunTask{{RunId: 99}, {RunId: 100}} + tasks := []jobs.RunTask{{RunId: 99}, {RunId: 100, AttemptNumber: 1}} finished := &jobs.Run{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated}, Tasks: tasks} assert.Equal(t, &mlflowIdentifiers{ExperimentID: "E1", RunID: "R1"}, mlflowIDs(ctx, w, finished)) @@ -105,6 +105,36 @@ func TestMLflowIDs(t *testing.T) { } assert.Nil(t, mlflowIDs(ctx, w, multiTask)) }) + + t.Run("a finished run links no older retry when a newer one's output can't be read", func(t *testing.T) { + // Given retry 0 logged to MLflow, but get-output fails for failingRunID. + finished := &jobs.Run{ + State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated}, + Tasks: []jobs.RunTask{{RunId: 98}, {RunId: 99, AttemptNumber: 1}, {RunId: 100, AttemptNumber: 2}}, + } + for _, failingRunID := range []string{"100", "99"} { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path != "/api/2.2/jobs/runs/get-output": + _, _ = w.Write([]byte(`{}`)) + case r.URL.Query().Get("run_id") == failingRunID: + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error_code":"PERMISSION_DENIED","message":"denied"}`)) + case r.URL.Query().Get("run_id") == "98": + _, _ = w.Write([]byte(`{"ai_runtime_task_output":{"mlflow_experiment_id":"E1","mlflow_run_id":"R0"}}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + + // When the views pick the run's MLflow run. + got := mlflowIDs(ctx, newTestWorkspaceClient(t, srv.URL), finished) + + // Then there is no link: the unread retry may have the newest MLflow run. + assert.Nil(t, got, "get-output fails for task run %s", failingRunID) + } + }) } func TestAiRuntimeTaskOutput(t *testing.T) { From 1f458c4b81b59d9c949e72f47e5fc44906437fab Mon Sep 17 00:00:00 2001 From: Pardis Beikzadeh Date: Thu, 12 Nov 2026 00:04:50 +0000 Subject: [PATCH 08/10] air: Name the older retry an MLflow link comes from, and pick it the same way in the list The list row linked the first retry's MLflow run while the info and logs panes linked the latest one that has a run. All of them now share one selection, and a link to an older retry says which, for example "(retry 5)". The list cache keeps the task runs this needs, so older cache entries are refreshed. Co-authored-by: Isaac --- .nextchanges/air/7456.md | 2 +- cmd/air/joblist.go | 12 ++++--- cmd/air/list.go | 21 +++++++------ cmd/air/list_cache.go | 27 +++++++++------- cmd/air/list_cache_test.go | 21 ++++++++----- cmd/air/list_detail.go | 2 +- cmd/air/list_detail_test.go | 62 ++++++++++++++++++++++++++++++++++++- cmd/air/list_index.go | 10 +++--- cmd/air/mlflow.go | 22 +++++++++++-- cmd/air/mlflow_test.go | 3 +- cmd/air/render.go | 2 +- 11 files changed, 138 insertions(+), 46 deletions(-) diff --git a/.nextchanges/air/7456.md b/.nextchanges/air/7456.md index d8aeec7ff6b..27b6c6551d4 100644 --- a/.nextchanges/air/7456.md +++ b/.nextchanges/air/7456.md @@ -1 +1 @@ -* Wrap long failure reasons and log lines in the `air list` info and logs panes, keep the MLflow link when a run's last retry failed before starting, and point `air logs` at earlier retries when the last one has no logs. ([#7456](https://github.com/databricks/cli/pull/7456)) +* Wrap long failure reasons and log lines in the `air list` info and logs panes, keep the MLflow link (labeled with its retry) when a run's last retry failed before starting, and point `air logs` at earlier retries when the last one has no logs. ([#7456](https://github.com/databricks/cli/pull/7456)) diff --git a/cmd/air/joblist.go b/cmd/air/joblist.go index dd786d8b496..680febb6338 100644 --- a/cmd/air/joblist.go +++ b/cmd/air/joblist.go @@ -53,12 +53,14 @@ func isSweep(r *jobs.Run) bool { return len(r.Tasks) > 0 && r.Tasks[0].ForEachTask != nil } -// taskRunID returns the run id of the AIR task, used to fetch its MLflow output. -func taskRunID(r *jobs.Run) int64 { - if len(r.Tasks) == 0 { - return 0 +// retryTasks returns the run's task runs with only the fields that pick its +// MLflow run (see mlflowIDsFromRetries), so the list cache stays small. +func retryTasks(r *jobs.Run) []jobs.RunTask { + tasks := make([]jobs.RunTask, len(r.Tasks)) + for i, t := range r.Tasks { + tasks[i] = jobs.RunTask{RunId: t.RunId, TaskKey: t.TaskKey, AttemptNumber: t.AttemptNumber} } - return r.Tasks[0].RunId + return tasks } // jobExperiment returns the run's MLflow experiment name (user-folder prefix diff --git a/cmd/air/list.go b/cmd/air/list.go index b80c644d17e..dcf40ad081c 100644 --- a/cmd/air/list.go +++ b/cmd/air/list.go @@ -59,12 +59,13 @@ type listRow struct { Accelerators string `json:"-"` } -// listedRun pairs a row with its parent and task run ids, so the MLflow link can -// be fetched after the run has been filtered in and cached afterward. +// listedRun pairs a row with its parent run id and task runs, so the MLflow link +// can be fetched after the run has been filtered in and cached afterward. type listedRun struct { - row listRow - runID int64 - taskRunID int64 + row listRow + runID int64 + tasks []jobs.RunTask // see retryTasks + terminal bool } // listQuery holds the resolved inputs to a runFetcher. @@ -310,7 +311,7 @@ func (s *jobsScanStrategy) next(want int) ([]listedRun, error) { if !s.filters.matches(run) { continue } - entries = append(entries, listedRun{row: buildListRow(run, s.host, s.workspaceID), runID: run.RunId, taskRunID: taskRunID(run)}) + entries = append(entries, listedRun{row: buildListRow(run, s.host, s.workspaceID), runID: run.RunId, tasks: retryTasks(run), terminal: isTerminal(run)}) } return entries, nil } @@ -343,13 +344,15 @@ func setMLflowLinks(ctx context.Context, w *databricks.WorkspaceClient, host str if entries[i].row.MLflowURL != "" && entries[i].row.MLflowURL != "-" { return nil } - ids := mlflowIDsForTask(ctx, w, entries[i].taskRunID) + ids := mlflowIDsForRetries(ctx, w, entries[i].tasks, entries[i].terminal) if ids == nil { return nil } entries[i].row.MLflowURL = mlflowLogsURL(host, ids) name := fetchMLflowRunName(ctx, w, ids.RunID) - entries[i].row.MLflowLabel = mlflowRunLabel(name, ids.RunID) + // Shorten the name, not the retry note, to fit the column. + note := ids.retryNote() + entries[i].row.MLflowLabel = truncate(mlflowRunLabel(name, ids.RunID), mlflowColWidth-len(note)) + note entries[i].row.ExperimentURL = mlflowExperimentURL(host, ids) // The ETA needs a progress-metric history fetch, so it's computed only // for running rows (the only ones that can have one). @@ -361,6 +364,6 @@ func setMLflowLinks(ctx context.Context, w *databricks.WorkspaceClient, host str return nil }) } - // mlflowIDsForTask never returns an error (it logs and yields nil), so Wait can't fail. + // mlflowIDsForRetries never returns an error (it logs and yields nil), so Wait can't fail. _ = g.Wait() } diff --git a/cmd/air/list_cache.go b/cmd/air/list_cache.go index a4d4d81ca06..722d1d95478 100644 --- a/cmd/air/list_cache.go +++ b/cmd/air/list_cache.go @@ -5,6 +5,7 @@ import ( "time" "github.com/databricks/cli/libs/cache" + "github.com/databricks/databricks-sdk-go/service/jobs" ) // The AiTrainingService index path caches hydrated terminal runs on disk: @@ -41,9 +42,11 @@ type cachedRun struct { RunURL string `json:"run_url"` ExperimentURL string `json:"experiment_url"` Accelerators string `json:"accelerators"` - TaskRunID int64 `json:"task_run_id"` Fields filterFields `json:"filter_fields"` SubmitTimeMs int64 `json:"submit_time_ms"` + + // Tasks are the run's task runs (see retryTasks), which pick its MLflow link. + Tasks []jobs.RunTask `json:"tasks"` } func (c cachedRun) toRow() listRow { @@ -55,13 +58,13 @@ func (c cachedRun) toRow() listRow { } } -func cachedRunFromRow(r listRow, taskRunID int64, fields filterFields, submitTimeMs int64) cachedRun { +func cachedRunFromRow(r listRow, tasks []jobs.RunTask, fields filterFields, submitTimeMs int64) cachedRun { return cachedRun{ RunID: r.RunID, RunName: r.RunName, User: r.User, Status: r.Status, StartedAt: r.StartedAt, IsSweep: r.IsSweep, Experiment: r.Experiment, Duration: r.Duration, MLflowURL: r.MLflowURL, MLflowLabel: r.MLflowLabel, RunURL: r.RunURL, ExperimentURL: r.ExperimentURL, Accelerators: r.Accelerators, - TaskRunID: taskRunID, Fields: fields, SubmitTimeMs: submitTimeMs, + Tasks: tasks, Fields: fields, SubmitTimeMs: submitTimeMs, } } @@ -71,19 +74,19 @@ func newListCache(ctx context.Context) *cache.Cache { return cache.NewCache(ctx, listCacheComponent, listCacheTTL, nil) } -// cachedRow returns the cached row, filter fields, and task run id for a run. -// Entries written before task ids were cached are treated as misses. -func cachedRow(ctx context.Context, c *cache.Cache, host string, runID int64) (listRow, filterFields, int64, bool) { +// cachedRow returns the cached row, filter fields, and task runs for a run. +// Entries written before task runs were cached are treated as misses. +func cachedRow(ctx context.Context, c *cache.Cache, host string, runID int64) (listRow, filterFields, []jobs.RunTask, bool) { entry, ok := cache.Get[cachedRun](ctx, c, listCacheKey{Host: host, RunID: runID}) - if !ok || entry.TaskRunID == 0 { - return listRow{}, filterFields{}, 0, false + if !ok || len(entry.Tasks) == 0 { + return listRow{}, filterFields{}, nil, false } - return entry.toRow(), entry.Fields, entry.TaskRunID, true + return entry.toRow(), entry.Fields, entry.Tasks, true } // putRow caches a terminal run's base row and filter fields under its submit time. -func putRow(ctx context.Context, c *cache.Cache, host string, runID, taskRunID, submitTimeMs int64, row listRow, fields filterFields) { - cache.Put(ctx, c, listCacheKey{Host: host, RunID: runID}, cachedRunFromRow(row, taskRunID, fields, submitTimeMs)) +func putRow(ctx context.Context, c *cache.Cache, host string, runID int64, tasks []jobs.RunTask, submitTimeMs int64, row listRow, fields filterFields) { + cache.Put(ctx, c, listCacheKey{Host: host, RunID: runID}, cachedRunFromRow(row, tasks, fields, submitTimeMs)) } // updateCachedRow replaces the display row while preserving cache metadata. @@ -93,5 +96,5 @@ func updateCachedRow(ctx context.Context, c *cache.Cache, host string, runID int if !ok { return } - cache.Put(ctx, c, key, cachedRunFromRow(row, entry.TaskRunID, entry.Fields, entry.SubmitTimeMs)) + cache.Put(ctx, c, key, cachedRunFromRow(row, entry.Tasks, entry.Fields, entry.SubmitTimeMs)) } diff --git a/cmd/air/list_cache_test.go b/cmd/air/list_cache_test.go index 820b7d1718f..42d54f5eed2 100644 --- a/cmd/air/list_cache_test.go +++ b/cmd/air/list_cache_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "testing" + "github.com/databricks/cli/libs/cache" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -21,20 +22,24 @@ func TestListCacheRoundTrip(t *testing.T) { row := listRow{RunID: "42", Experiment: "exp", Status: "SUCCESS"} fields := filterFields{Experiment: "exp", GPUType: "GPU_1xA10", GPUCount: 1} - putRow(ctx, c, "https://host.test", 42, 420, 1700000000000, row, fields) + tasks := []jobs.RunTask{{RunId: 420, TaskKey: "train"}, {RunId: 421, TaskKey: "train", AttemptNumber: 1}} + putRow(ctx, c, "https://host.test", 42, tasks, 1700000000000, row, fields) - got, gotFields, taskRunID, ok := cachedRow(ctx, c, "https://host.test", 42) + got, gotFields, gotTasks, ok := cachedRow(ctx, c, "https://host.test", 42) require.True(t, ok, "hit after write") assert.Equal(t, row, got) assert.Equal(t, fields, gotFields) - assert.Equal(t, int64(420), taskRunID) + for i := range gotTasks { + gotTasks[i].ForceSendFields = nil // set by the JSON round trip + } + assert.Equal(t, tasks, gotTasks) // Different host is a different key. _, _, _, ok = cachedRow(ctx, c, "https://other.test", 42) assert.False(t, ok) - // Entries from the previous cache format have no task run id and must be refreshed. - putRow(ctx, c, "https://host.test", 43, 0, 1700000000000, listRow{RunID: "43"}, filterFields{}) + // Entries from previous cache formats have no task runs and must be refreshed. + cache.Put(ctx, c, listCacheKey{Host: "https://host.test", RunID: 43}, map[string]any{"run_id": "43", "task_run_id": 430}) _, _, _, ok = cachedRow(ctx, c, "https://host.test", 43) assert.False(t, ok) } @@ -48,7 +53,7 @@ func TestIndexStrategyServesCachedRowWithoutFetch(t *testing.T) { // Pre-seed the cache for run 7 so hydration should skip runs/get entirely. ctx := t.Context() - putRow(ctx, newListCache(ctx), host, 7, 70, 1000_000, listRow{RunID: "7", Status: "SUCCESS"}, filterFields{}) + putRow(ctx, newListCache(ctx), host, 7, []jobs.RunTask{{RunId: 70}}, 1000_000, listRow{RunID: "7", Status: "SUCCESS"}, filterFields{}) f := newRunFetcher(ctx, newTestWorkspaceClient(t, host), listQuery{ userFilter: "me@example.com", currentUser: "me@example.com", limit: 10, @@ -69,7 +74,7 @@ func TestIndexStrategyFiltersCachedRow(t *testing.T) { // Cache hit under experiment "bar" must be filtered out by an experiment=foo query. ctx := t.Context() - putRow(ctx, newListCache(ctx), host, 7, 70, 1000_000, + putRow(ctx, newListCache(ctx), host, 7, []jobs.RunTask{{RunId: 70}}, 1000_000, listRow{RunID: "7", Status: "SUCCESS", Experiment: "bar"}, filterFields{Experiment: "bar"}) @@ -91,7 +96,7 @@ func TestIndexStrategyServesMatchingCachedRow(t *testing.T) { host := srv.URL ctx := t.Context() - putRow(ctx, newListCache(ctx), host, 7, 70, 1000_000, + putRow(ctx, newListCache(ctx), host, 7, []jobs.RunTask{{RunId: 70}}, 1000_000, listRow{RunID: "7", Status: "SUCCESS", Experiment: "foo"}, filterFields{Experiment: "foo"}) diff --git a/cmd/air/list_detail.go b/cmd/air/list_detail.go index 62627030605..96e50728dc0 100644 --- a/cmd/air/list_detail.go +++ b/cmd/air/list_detail.go @@ -62,7 +62,7 @@ func runLogsSnapshot(ctx context.Context, w *databricks.WorkspaceClient, runID i return "", fmt.Errorf("failed to fetch logs: %w", err) } if ids := mlflowIDs(ctx, w, run); ids != nil { - fmt.Fprintf(&buf, "\nMLflow URL: %s\n", mlflowLogsURL(w.Config.Host, ids)) + fmt.Fprintf(&buf, "\nMLflow URL%s: %s\n", ids.retryNote(), mlflowLogsURL(w.Config.Host, ids)) } return buf.String(), nil } diff --git a/cmd/air/list_detail_test.go b/cmd/air/list_detail_test.go index a1678e130fe..178d1f8bc34 100644 --- a/cmd/air/list_detail_test.go +++ b/cmd/air/list_detail_test.go @@ -1,11 +1,14 @@ package aircmd import ( + "encoding/json" "net/http" "net/http/httptest" "strings" "testing" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -33,5 +36,62 @@ func TestRunLogsSnapshotFailedRetryWithoutLogs(t *testing.T) { require.NoError(t, err) assert.Equal(t, "No logs available for run 123. Run terminated in state FAILED: Out of capacity. "+ "Earlier retries may have logs; pass --retry N, from 0 to 1, to see them\n"+ - "\nMLflow URL: "+srv.URL+"/ml/experiments/E1/runs/R1/artifacts/logs/node_0\n", got) + "\nMLflow URL (retry 0): "+srv.URL+"/ml/experiments/E1/runs/R1/artifacts/logs/node_0\n", got) +} + +func TestListRowAndPanesLinkTheSameOlderRetry(t *testing.T) { + // Given a failed run whose retries 0 and 1 logged to MLflow and whose last + // retry failed before it started. + t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir()) + run := indexRun(7, 1000_000) + run.State.ResultState = jobs.RunResultStateFailed + task := run.Tasks[0] + run.Tasks = nil + for attempt, taskRunID := range []int64{70, 71, 72} { + task.RunId, task.AttemptNumber = taskRunID, attempt + run.Tasks = append(run.Tasks, task) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case aiTrainingWorkflowsPath: + _, _ = w.Write([]byte(`{"training_workflows": [{"job_run_id": "7", "submit_time": {"seconds": 1000}}]}`)) + case "/api/2.2/jobs/runs/get": + _ = json.NewEncoder(w).Encode(run) + case "/api/2.2/jobs/runs/get-output": + switch r.URL.Query().Get("run_id") { + case "70": + _, _ = w.Write([]byte(`{"ai_runtime_task_output": {"mlflow_experiment_id": "E1", "mlflow_run_id": "R0"}}`)) + case "71": + _, _ = w.Write([]byte(`{"ai_runtime_task_output": {"mlflow_experiment_id": "E1", "mlflow_run_id": "R1"}}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + case "/api/2.0/mlflow/runs/get": + _, _ = w.Write([]byte(`{"run": {"info": {"run_name": "train-` + r.URL.Query().Get("run_id") + `"}}}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + ctx := cmdio.MockDiscard(t.Context()) + w := newTestWorkspaceClient(t, srv.URL) + wantURL := srv.URL + "/ml/experiments/E1/runs/R1/artifacts/logs/node_0" + + // When the list, the info pane and the logs pane render the run. + rows, err := newRunFetcher(ctx, w, listQuery{ + userFilter: "me@example.com", currentUser: "me@example.com", limit: 10, fetchMLflow: true, + }).next(10) + require.NoError(t, err) + info, err := runDetailText(ctx, w, 7) + require.NoError(t, err) + logs, err := runLogsSnapshot(ctx, w, 7) + require.NoError(t, err) + + // Then each links retry 1's MLflow run and names that retry. + require.Len(t, rows, 1) + assert.Equal(t, wantURL, rows[0].MLflowURL) + assert.Equal(t, "train-R1 (retry 1)", rows[0].MLflowLabel) + assert.Contains(t, info, "train-R1 (retry 1)") + assert.Contains(t, info, "MLflow URL: "+srv.URL+"/ml/experiments/E1/runs/R1\n") + assert.Contains(t, logs, "\nMLflow URL (retry 1): "+wantURL+"\n") } diff --git a/cmd/air/list_index.go b/cmd/air/list_index.go index 52a70173a97..3bd1839f7b6 100644 --- a/cmd/air/list_index.go +++ b/cmd/air/list_index.go @@ -106,9 +106,9 @@ func (s *indexStrategy) hydrate(ids []int64) ([]listedRun, error) { // The cache key excludes the filter, so a cached row is still run through the // active filter; a non-matching hit is dropped, not re-fetched. for _, id := range ids { - if row, fields, taskRunID, ok := cachedRow(s.ctx, s.cache, host, id); ok { + if row, fields, tasks, ok := cachedRow(s.ctx, s.cache, host, id); ok { if s.filters.matchesFields(fields) { - rows = append(rows, listedRun{row: row, runID: id, taskRunID: taskRunID}) + rows = append(rows, listedRun{row: row, runID: id, tasks: tasks, terminal: true}) } continue } @@ -125,11 +125,11 @@ func (s *indexStrategy) hydrate(ids []int64) ([]listedRun, error) { continue } row := buildListRow(run, s.host, s.workspaceID) - taskRunID := taskRunID(run) - rows = append(rows, listedRun{row: row, runID: run.RunId, taskRunID: taskRunID}) + tasks := retryTasks(run) + rows = append(rows, listedRun{row: row, runID: run.RunId, tasks: tasks, terminal: isTerminal(run)}) if isTerminal(run) { start, _ := jobTiming(run) - putRow(s.ctx, s.cache, host, run.RunId, taskRunID, start, row, fields) + putRow(s.ctx, s.cache, host, run.RunId, tasks, start, row, fields) } } diff --git a/cmd/air/mlflow.go b/cmd/air/mlflow.go index 961b8d82099..d05c9746b19 100644 --- a/cmd/air/mlflow.go +++ b/cmd/air/mlflow.go @@ -16,13 +16,30 @@ import ( type mlflowIdentifiers struct { ExperimentID string RunID string + // OlderRetry is the retry the IDs came from when it isn't the run's latest + // (see mlflowIDsFromRetries), else nil. + OlderRetry *int +} + +// retryNote is " (retry N)" when ids come from an older retry, else "". +func (ids *mlflowIdentifiers) retryNote() string { + if ids.OlderRetry == nil { + return "" + } + return fmt.Sprintf(" (retry %d)", *ids.OlderRetry) } // mlflowIDs fetches the MLflow IDs a run's views link to (see // mlflowIDsFromRetries). Returns nil if they can't be obtained. func mlflowIDs(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run) *mlflowIdentifiers { - latest, err := latestTaskOutput(ctx, w, run.Tasks) - return mlflowIDsFromRetries(ctx, w, run.Tasks, isTerminal(run), latest, err) + return mlflowIDsForRetries(ctx, w, run.Tasks, isTerminal(run)) +} + +// mlflowIDsForRetries is mlflowIDs given only the task runs and whether the run +// has finished, which is what a cached list row keeps. +func mlflowIDsForRetries(ctx context.Context, w *databricks.WorkspaceClient, tasks []jobs.RunTask, terminal bool) *mlflowIdentifiers { + latest, err := latestTaskOutput(ctx, w, tasks) + return mlflowIDsFromRetries(ctx, w, tasks, terminal, latest, err) } // mlflowIDsFromRetries returns the IDs from latest (the latest retry's output), @@ -51,6 +68,7 @@ func mlflowIDsFromRetries(ctx context.Context, w *databricks.WorkspaceClient, ta return nil } if ids := mlflowIDsFromOutput(out); ids != nil { + ids.OlderRetry = &task.AttemptNumber return ids } } diff --git a/cmd/air/mlflow_test.go b/cmd/air/mlflow_test.go index b08fa971465..930f0b55af1 100644 --- a/cmd/air/mlflow_test.go +++ b/cmd/air/mlflow_test.go @@ -92,7 +92,8 @@ func TestMLflowIDs(t *testing.T) { w := newTestWorkspaceClient(t, srv.URL) tasks := []jobs.RunTask{{RunId: 99}, {RunId: 100, AttemptNumber: 1}} finished := &jobs.Run{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated}, Tasks: tasks} - assert.Equal(t, &mlflowIdentifiers{ExperimentID: "E1", RunID: "R1"}, mlflowIDs(ctx, w, finished)) + retry0 := 0 + assert.Equal(t, &mlflowIdentifiers{ExperimentID: "E1", RunID: "R1", OlderRetry: &retry0}, mlflowIDs(ctx, w, finished)) // A run still starting its next retry must not link to the previous attempt. running := &jobs.Run{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, Tasks: tasks} diff --git a/cmd/air/render.go b/cmd/air/render.go index f7a96f27b26..3985ab57588 100644 --- a/cmd/air/render.go +++ b/cmd/air/render.go @@ -124,7 +124,7 @@ func renderRunText(ctx context.Context, out io.Writer, w *databricks.WorkspaceCl } if ids != nil { - view.mlflowLabel = mlflowRunLabel(fetchMLflowRunName(ctx, w, ids.RunID), ids.RunID) + view.mlflowLabel = mlflowRunLabel(fetchMLflowRunName(ctx, w, ids.RunID), ids.RunID) + ids.retryNote() view.mlflowURL = mlflowRunURL(w.Config.Host, ids) } From 933eff9ddc29e9cfb55328d9ebd8501b18b8a364 Mon Sep 17 00:00:00 2001 From: Pardis Beikzadeh Date: Thu, 12 Nov 2026 00:06:08 +0000 Subject: [PATCH 09/10] air: Reword the earlier-retries hint to "pass --retry N (0 through M)" Co-authored-by: Isaac --- acceptance/air/logs-failed-retry/output.txt | 2 +- cmd/air/list_detail_test.go | 2 +- cmd/air/logstream_support.go | 2 +- cmd/air/logstream_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/air/logs-failed-retry/output.txt b/acceptance/air/logs-failed-retry/output.txt index a135296c266..197f261ef5a 100644 --- a/acceptance/air/logs-failed-retry/output.txt +++ b/acceptance/air/logs-failed-retry/output.txt @@ -2,6 +2,6 @@ === logs of a failed retry with no logs point to earlier retries >>> [CLI] air logs 123 -No logs available for run 123. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs; pass --retry N, from 0 to 1, to see them +No logs available for run 123. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs; pass --retry N (0 through 1) to see them Exit code: 1 diff --git a/cmd/air/list_detail_test.go b/cmd/air/list_detail_test.go index 178d1f8bc34..d5b005a5941 100644 --- a/cmd/air/list_detail_test.go +++ b/cmd/air/list_detail_test.go @@ -35,7 +35,7 @@ func TestRunLogsSnapshotFailedRetryWithoutLogs(t *testing.T) { got, err := runLogsSnapshot(t.Context(), newTestWorkspaceClient(t, srv.URL), 123) require.NoError(t, err) assert.Equal(t, "No logs available for run 123. Run terminated in state FAILED: Out of capacity. "+ - "Earlier retries may have logs; pass --retry N, from 0 to 1, to see them\n"+ + "Earlier retries may have logs; pass --retry N (0 through 1) to see them\n"+ "\nMLflow URL (retry 0): "+srv.URL+"/ml/experiments/E1/runs/R1/artifacts/logs/node_0\n", got) } diff --git a/cmd/air/logstream_support.go b/cmd/air/logstream_support.go index 9817445decc..cb7a8d2d6d3 100644 --- a/cmd/air/logstream_support.go +++ b/cmd/air/logstream_support.go @@ -238,7 +238,7 @@ func emitNoLogs(out io.Writer, req logRequest, status logRunStatus) { if req.attempt < 0 && status.terminal() && status.latestAttempt > 0 { retry := "0" if status.latestAttempt > 1 { - retry = fmt.Sprintf("N, from 0 to %d,", status.latestAttempt-1) + retry = fmt.Sprintf("N (0 through %d)", status.latestAttempt-1) } msg = fmt.Sprintf("%s. Earlier retries may have logs; pass --retry %s to see them", strings.TrimRight(msg, "."), retry) } diff --git a/cmd/air/logstream_test.go b/cmd/air/logstream_test.go index 1d76c07d9fb..b1006d9b49c 100644 --- a/cmd/air/logstream_test.go +++ b/cmd/air/logstream_test.go @@ -548,7 +548,7 @@ func TestEmitNoLogs(t *testing.T) { name: "latest retry of a terminal run", req: logRequest{attempt: -1}, status: logRunStatus{lifeCycleState: "TERMINATED", resultState: "FAILED", stateMessage: "Out of capacity.", latestAttempt: 2}, - want: "No logs available for run 7. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs; pass --retry N, from 0 to 1, to see them\n", + want: "No logs available for run 7. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs; pass --retry N (0 through 1) to see them\n", }, { name: "latest retry of a terminal run with one earlier retry", From 6d6aaa95f055b433b2d4a1ffc8c4220314207432 Mon Sep 17 00:00:00 2001 From: Pardis Beikzadeh Date: Thu, 12 Nov 2026 21:36:05 +0000 Subject: [PATCH 10/10] air: Expect the MLflow Logs page URL in the older-retry link tests #7491 moved MLflow log links from Artifacts to the Logs page. Co-authored-by: Isaac --- cmd/air/list_detail_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/air/list_detail_test.go b/cmd/air/list_detail_test.go index d5b005a5941..8cbe2c8d3b6 100644 --- a/cmd/air/list_detail_test.go +++ b/cmd/air/list_detail_test.go @@ -36,7 +36,7 @@ func TestRunLogsSnapshotFailedRetryWithoutLogs(t *testing.T) { require.NoError(t, err) assert.Equal(t, "No logs available for run 123. Run terminated in state FAILED: Out of capacity. "+ "Earlier retries may have logs; pass --retry N (0 through 1) to see them\n"+ - "\nMLflow URL (retry 0): "+srv.URL+"/ml/experiments/E1/runs/R1/artifacts/logs/node_0\n", got) + "\nMLflow URL (retry 0): "+srv.URL+"/ml/experiments/E1/runs/R1/logs\n", got) } func TestListRowAndPanesLinkTheSameOlderRetry(t *testing.T) { @@ -75,7 +75,7 @@ func TestListRowAndPanesLinkTheSameOlderRetry(t *testing.T) { t.Cleanup(srv.Close) ctx := cmdio.MockDiscard(t.Context()) w := newTestWorkspaceClient(t, srv.URL) - wantURL := srv.URL + "/ml/experiments/E1/runs/R1/artifacts/logs/node_0" + wantURL := srv.URL + "/ml/experiments/E1/runs/R1/logs" // When the list, the info pane and the logs pane render the run. rows, err := newRunFetcher(ctx, w, listQuery{