diff --git a/.nextchanges/air/7456.md b/.nextchanges/air/7456.md new file mode 100644 index 00000000000..27b6c6551d4 --- /dev/null +++ b/.nextchanges/air/7456.md @@ -0,0 +1 @@ +* Wrap long failure reasons and log lines in the `air list` info and logs panes, keep the MLflow link (labeled with its retry) when a run's last retry failed before starting, and point `air logs` at earlier retries when the last one has no logs. ([#7456](https://github.com/databricks/cli/pull/7456)) diff --git a/acceptance/air/logs-failed-retry/out.test.toml b/acceptance/air/logs-failed-retry/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/air/logs-failed-retry/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/air/logs-failed-retry/output.txt b/acceptance/air/logs-failed-retry/output.txt new file mode 100644 index 00000000000..197f261ef5a --- /dev/null +++ b/acceptance/air/logs-failed-retry/output.txt @@ -0,0 +1,7 @@ + +=== logs of a failed retry with no logs point to earlier retries + +>>> [CLI] air logs 123 +No logs available for run 123. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs; pass --retry N (0 through 1) to see them + +Exit code: 1 diff --git a/acceptance/air/logs-failed-retry/script b/acceptance/air/logs-failed-retry/script new file mode 100644 index 00000000000..f76fb66a16f --- /dev/null +++ b/acceptance/air/logs-failed-retry/script @@ -0,0 +1,2 @@ +title "logs of a failed retry with no logs point to earlier retries" +errcode trace $CLI air logs 123 diff --git a/acceptance/air/logs-failed-retry/test.toml b/acceptance/air/logs-failed-retry/test.toml new file mode 100644 index 00000000000..d9f7cf4ae81 --- /dev/null +++ b/acceptance/air/logs-failed-retry/test.toml @@ -0,0 +1,31 @@ +# The SDK occasionally probes host reachability with a HEAD request; stub it so +# the test is deterministic. +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +# A failed run whose second and third attempts failed before they started. +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get" +Response.Body = ''' +{ + "run_id": 123, + "start_time": 1700000000000, + "end_time": 1700000012000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "Out of capacity"}, + "tasks": [ + {"task_key": "train", "run_id": 456, "attempt_number": 0}, + {"task_key": "train", "run_id": 457, "attempt_number": 1}, + {"task_key": "train", "run_id": 458, "attempt_number": 2} + ] +} +''' + +# Neither Bricklens nor MLflow has logs for the latest attempt. +[[Server]] +Pattern = "GET /api/2.0/ai-training/workflows/by-run-id/123/logs" +Response.Body = '{"log_records": []}' + +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get-output" +Response.Body = '{}' diff --git a/cmd/air/get.go b/cmd/air/get.go index aa20e3b16a2..18967532981 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -167,8 +167,8 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, data := buildGetData(run) data.DashboardURL = dashboardURL(w.Config.Host, runID, workspaceID) - taskOutput := aiRuntimeTaskOutput(ctx, w, run) - ids := mlflowIDsFromOutput(taskOutput) + taskOutput, taskOutputErr := latestTaskOutput(ctx, w, run.Tasks) + ids := mlflowIDsFromRetries(ctx, w, run.Tasks, isTerminal(run), taskOutput, taskOutputErr) if ids != nil { url := mlflowLogsURL(w.Config.Host, ids) data.MLflowURL = &url diff --git a/cmd/air/joblist.go b/cmd/air/joblist.go index dd786d8b496..680febb6338 100644 --- a/cmd/air/joblist.go +++ b/cmd/air/joblist.go @@ -53,12 +53,14 @@ func isSweep(r *jobs.Run) bool { return len(r.Tasks) > 0 && r.Tasks[0].ForEachTask != nil } -// taskRunID returns the run id of the AIR task, used to fetch its MLflow output. -func taskRunID(r *jobs.Run) int64 { - if len(r.Tasks) == 0 { - return 0 +// retryTasks returns the run's task runs with only the fields that pick its +// MLflow run (see mlflowIDsFromRetries), so the list cache stays small. +func retryTasks(r *jobs.Run) []jobs.RunTask { + tasks := make([]jobs.RunTask, len(r.Tasks)) + for i, t := range r.Tasks { + tasks[i] = jobs.RunTask{RunId: t.RunId, TaskKey: t.TaskKey, AttemptNumber: t.AttemptNumber} } - return r.Tasks[0].RunId + return tasks } // jobExperiment returns the run's MLflow experiment name (user-folder prefix diff --git a/cmd/air/list.go b/cmd/air/list.go index 61cfe3d9b3b..cc2b27c99d4 100644 --- a/cmd/air/list.go +++ b/cmd/air/list.go @@ -59,12 +59,13 @@ type listRow struct { Accelerators string `json:"-"` } -// listedRun pairs a row with its parent and task run ids, so the MLflow link can -// be fetched after the run has been filtered in and cached afterward. +// listedRun pairs a row with its parent run id and task runs, so the MLflow link +// can be fetched after the run has been filtered in and cached afterward. type listedRun struct { - row listRow - runID int64 - taskRunID int64 + row listRow + runID int64 + tasks []jobs.RunTask // see retryTasks + terminal bool } // listQuery holds the resolved inputs to a runFetcher. @@ -310,7 +311,7 @@ func (s *jobsScanStrategy) next(want int) ([]listedRun, error) { if !s.filters.matches(run) { continue } - entries = append(entries, listedRun{row: buildListRow(run, s.host, s.workspaceID), runID: run.RunId, taskRunID: taskRunID(run)}) + entries = append(entries, listedRun{row: buildListRow(run, s.host, s.workspaceID), runID: run.RunId, tasks: retryTasks(run), terminal: isTerminal(run)}) } return entries, nil } @@ -343,13 +344,15 @@ func setMLflowLinks(ctx context.Context, w *databricks.WorkspaceClient, host str if entries[i].row.MLflowURL != "" && entries[i].row.MLflowURL != "-" { return nil } - ids := mlflowIDsForTask(ctx, w, entries[i].taskRunID) + ids := mlflowIDsForRetries(ctx, w, entries[i].tasks, entries[i].terminal) if ids == nil { return nil } entries[i].row.MLflowURL = mlflowLogsURL(host, ids) name := fetchMLflowRunName(ctx, w, ids.RunID) - entries[i].row.MLflowLabel = mlflowRunLabel(name, ids.RunID) + // Shorten the name, not the retry note, to fit the column. + note := ids.retryNote() + entries[i].row.MLflowLabel = truncate(mlflowRunLabel(name, ids.RunID), mlflowColWidth-len(note)) + note entries[i].row.ExperimentURL = mlflowExperimentURL(host, ids) // The ETA needs a progress-metric history fetch, so it's computed only // for running rows (the only ones that can have one). @@ -361,6 +364,6 @@ func setMLflowLinks(ctx context.Context, w *databricks.WorkspaceClient, host str return nil }) } - // mlflowIDsForTask never returns an error (it logs and yields nil), so Wait can't fail. + // mlflowIDsForRetries never returns an error (it logs and yields nil), so Wait can't fail. _ = g.Wait() } diff --git a/cmd/air/list_cache.go b/cmd/air/list_cache.go index a4d4d81ca06..722d1d95478 100644 --- a/cmd/air/list_cache.go +++ b/cmd/air/list_cache.go @@ -5,6 +5,7 @@ import ( "time" "github.com/databricks/cli/libs/cache" + "github.com/databricks/databricks-sdk-go/service/jobs" ) // The AiTrainingService index path caches hydrated terminal runs on disk: @@ -41,9 +42,11 @@ type cachedRun struct { RunURL string `json:"run_url"` ExperimentURL string `json:"experiment_url"` Accelerators string `json:"accelerators"` - TaskRunID int64 `json:"task_run_id"` Fields filterFields `json:"filter_fields"` SubmitTimeMs int64 `json:"submit_time_ms"` + + // Tasks are the run's task runs (see retryTasks), which pick its MLflow link. + Tasks []jobs.RunTask `json:"tasks"` } func (c cachedRun) toRow() listRow { @@ -55,13 +58,13 @@ func (c cachedRun) toRow() listRow { } } -func cachedRunFromRow(r listRow, taskRunID int64, fields filterFields, submitTimeMs int64) cachedRun { +func cachedRunFromRow(r listRow, tasks []jobs.RunTask, fields filterFields, submitTimeMs int64) cachedRun { return cachedRun{ RunID: r.RunID, RunName: r.RunName, User: r.User, Status: r.Status, StartedAt: r.StartedAt, IsSweep: r.IsSweep, Experiment: r.Experiment, Duration: r.Duration, MLflowURL: r.MLflowURL, MLflowLabel: r.MLflowLabel, RunURL: r.RunURL, ExperimentURL: r.ExperimentURL, Accelerators: r.Accelerators, - TaskRunID: taskRunID, Fields: fields, SubmitTimeMs: submitTimeMs, + Tasks: tasks, Fields: fields, SubmitTimeMs: submitTimeMs, } } @@ -71,19 +74,19 @@ func newListCache(ctx context.Context) *cache.Cache { return cache.NewCache(ctx, listCacheComponent, listCacheTTL, nil) } -// cachedRow returns the cached row, filter fields, and task run id for a run. -// Entries written before task ids were cached are treated as misses. -func cachedRow(ctx context.Context, c *cache.Cache, host string, runID int64) (listRow, filterFields, int64, bool) { +// cachedRow returns the cached row, filter fields, and task runs for a run. +// Entries written before task runs were cached are treated as misses. +func cachedRow(ctx context.Context, c *cache.Cache, host string, runID int64) (listRow, filterFields, []jobs.RunTask, bool) { entry, ok := cache.Get[cachedRun](ctx, c, listCacheKey{Host: host, RunID: runID}) - if !ok || entry.TaskRunID == 0 { - return listRow{}, filterFields{}, 0, false + if !ok || len(entry.Tasks) == 0 { + return listRow{}, filterFields{}, nil, false } - return entry.toRow(), entry.Fields, entry.TaskRunID, true + return entry.toRow(), entry.Fields, entry.Tasks, true } // putRow caches a terminal run's base row and filter fields under its submit time. -func putRow(ctx context.Context, c *cache.Cache, host string, runID, taskRunID, submitTimeMs int64, row listRow, fields filterFields) { - cache.Put(ctx, c, listCacheKey{Host: host, RunID: runID}, cachedRunFromRow(row, taskRunID, fields, submitTimeMs)) +func putRow(ctx context.Context, c *cache.Cache, host string, runID int64, tasks []jobs.RunTask, submitTimeMs int64, row listRow, fields filterFields) { + cache.Put(ctx, c, listCacheKey{Host: host, RunID: runID}, cachedRunFromRow(row, tasks, fields, submitTimeMs)) } // updateCachedRow replaces the display row while preserving cache metadata. @@ -93,5 +96,5 @@ func updateCachedRow(ctx context.Context, c *cache.Cache, host string, runID int if !ok { return } - cache.Put(ctx, c, key, cachedRunFromRow(row, entry.TaskRunID, entry.Fields, entry.SubmitTimeMs)) + cache.Put(ctx, c, key, cachedRunFromRow(row, entry.Tasks, entry.Fields, entry.SubmitTimeMs)) } diff --git a/cmd/air/list_cache_test.go b/cmd/air/list_cache_test.go index afeb2a2e490..8e4e9be4fd0 100644 --- a/cmd/air/list_cache_test.go +++ b/cmd/air/list_cache_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "testing" + "github.com/databricks/cli/libs/cache" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -21,20 +22,24 @@ func TestListCacheRoundTrip(t *testing.T) { row := listRow{RunID: "42", Experiment: "exp", Status: "SUCCESS"} fields := filterFields{Experiment: "exp", GPUType: "GPU_1xA10", GPUCount: 1} - putRow(ctx, c, "https://host.test", 42, 420, 1700000000000, row, fields) + tasks := []jobs.RunTask{{RunId: 420, TaskKey: "train"}, {RunId: 421, TaskKey: "train", AttemptNumber: 1}} + putRow(ctx, c, "https://host.test", 42, tasks, 1700000000000, row, fields) - got, gotFields, taskRunID, ok := cachedRow(ctx, c, "https://host.test", 42) + got, gotFields, gotTasks, ok := cachedRow(ctx, c, "https://host.test", 42) require.True(t, ok, "hit after write") assert.Equal(t, row, got) assert.Equal(t, fields, gotFields) - assert.Equal(t, int64(420), taskRunID) + for i := range gotTasks { + gotTasks[i].ForceSendFields = nil // set by the JSON round trip + } + assert.Equal(t, tasks, gotTasks) // Different host is a different key. _, _, _, ok = cachedRow(ctx, c, "https://other.test", 42) assert.False(t, ok) - // Entries from the previous cache format have no task run id and must be refreshed. - putRow(ctx, c, "https://host.test", 43, 0, 1700000000000, listRow{RunID: "43"}, filterFields{}) + // Entries from previous cache formats have no task runs and must be refreshed. + cache.Put(ctx, c, listCacheKey{Host: "https://host.test", RunID: 43}, map[string]any{"run_id": "43", "task_run_id": 430}) _, _, _, ok = cachedRow(ctx, c, "https://host.test", 43) assert.False(t, ok) } @@ -48,7 +53,7 @@ func TestIndexStrategyServesCachedRowWithoutFetch(t *testing.T) { // Pre-seed the cache for run 7 so hydration should skip runs/get entirely. ctx := t.Context() - putRow(ctx, newListCache(ctx), host, 7, 70, 1000_000, listRow{RunID: "7", Status: "SUCCESS"}, filterFields{}) + putRow(ctx, newListCache(ctx), host, 7, []jobs.RunTask{{RunId: 70}}, 1000_000, listRow{RunID: "7", Status: "SUCCESS"}, filterFields{}) f := newRunFetcher(ctx, newTestWorkspaceClient(t, host), listQuery{ userFilter: "me@example.com", currentUser: "me@example.com", limit: 10, @@ -69,7 +74,7 @@ func TestIndexStrategyFiltersCachedRow(t *testing.T) { // Cache hit under experiment "bar" must be filtered out by an experiment=foo query. ctx := t.Context() - putRow(ctx, newListCache(ctx), host, 7, 70, 1000_000, + putRow(ctx, newListCache(ctx), host, 7, []jobs.RunTask{{RunId: 70}}, 1000_000, listRow{RunID: "7", Status: "SUCCESS", Experiment: "bar"}, filterFields{Experiment: "bar"}) @@ -91,7 +96,7 @@ func TestIndexStrategyServesMatchingCachedRow(t *testing.T) { host := srv.URL ctx := t.Context() - putRow(ctx, newListCache(ctx), host, 7, 70, 1000_000, + putRow(ctx, newListCache(ctx), host, 7, []jobs.RunTask{{RunId: 70}}, 1000_000, listRow{RunID: "7", Status: "SUCCESS", Experiment: "foo"}, filterFields{Experiment: "foo"}) diff --git a/cmd/air/list_detail.go b/cmd/air/list_detail.go index c0eca25b13f..96e50728dc0 100644 --- a/cmd/air/list_detail.go +++ b/cmd/air/list_detail.go @@ -39,15 +39,16 @@ func runDetailText(ctx context.Context, w *databricks.WorkspaceClient, runID int } // runLogsSnapshot fetches a one-shot tail of a run's logs into a string, for the -// list picker's detail pane. +// list picker's detail pane, followed by the run's MLflow logs link. func runLogsSnapshot(ctx context.Context, w *databricks.WorkspaceClient, runID int64) (string, error) { - status, err := resolveRunStatus(ctx, w, runID) + run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: runID}) if err != nil { if errors.Is(err, apierr.ErrResourceDoesNotExist) { return "", fmt.Errorf("run %d not found", runID) } return "", fmt.Errorf("failed to fetch run status: %w", err) } + status := projectRunStatus(run) req := logRequest{ runID: runID, @@ -60,5 +61,8 @@ func runLogsSnapshot(ctx context.Context, w *databricks.WorkspaceClient, runID i if _, err := fetchLogs(ctx, w, &buf, req, status); err != nil { return "", fmt.Errorf("failed to fetch logs: %w", err) } + if ids := mlflowIDs(ctx, w, run); ids != nil { + fmt.Fprintf(&buf, "\nMLflow URL%s: %s\n", ids.retryNote(), mlflowLogsURL(w.Config.Host, ids)) + } return buf.String(), nil } diff --git a/cmd/air/list_detail_test.go b/cmd/air/list_detail_test.go new file mode 100644 index 00000000000..8cbe2c8d3b6 --- /dev/null +++ b/cmd/air/list_detail_test.go @@ -0,0 +1,97 @@ +package aircmd + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRunLogsSnapshotFailedRetryWithoutLogs(t *testing.T) { + // Only attempt 0 ran: retries 1 and 2 failed before they started, with no + // logs or MLflow output of their own. The pane still links the MLflow run + // that attempt 0 logged to. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/logs"): + _, _ = w.Write([]byte(`{"log_records": []}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get": + _, _ = w.Write([]byte(`{"run_id": 123, "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "Out of capacity"}, + "tasks": [{"run_id": 456, "attempt_number": 0}, {"run_id": 457, "attempt_number": 1}, {"run_id": 458, "attempt_number": 2}]}`)) + case r.URL.Path == "/api/2.2/jobs/runs/get-output" && r.URL.Query().Get("run_id") == "456": + _, _ = w.Write([]byte(`{"ai_runtime_task_output": {"mlflow_experiment_id": "E1", "mlflow_run_id": "R1"}}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + + got, err := runLogsSnapshot(t.Context(), newTestWorkspaceClient(t, srv.URL), 123) + require.NoError(t, err) + assert.Equal(t, "No logs available for run 123. Run terminated in state FAILED: Out of capacity. "+ + "Earlier retries may have logs; pass --retry N (0 through 1) to see them\n"+ + "\nMLflow URL (retry 0): "+srv.URL+"/ml/experiments/E1/runs/R1/logs\n", got) +} + +func TestListRowAndPanesLinkTheSameOlderRetry(t *testing.T) { + // Given a failed run whose retries 0 and 1 logged to MLflow and whose last + // retry failed before it started. + t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir()) + run := indexRun(7, 1000_000) + run.State.ResultState = jobs.RunResultStateFailed + task := run.Tasks[0] + run.Tasks = nil + for attempt, taskRunID := range []int64{70, 71, 72} { + task.RunId, task.AttemptNumber = taskRunID, attempt + run.Tasks = append(run.Tasks, task) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case aiTrainingWorkflowsPath: + _, _ = w.Write([]byte(`{"training_workflows": [{"job_run_id": "7", "submit_time": {"seconds": 1000}}]}`)) + case "/api/2.2/jobs/runs/get": + _ = json.NewEncoder(w).Encode(run) + case "/api/2.2/jobs/runs/get-output": + switch r.URL.Query().Get("run_id") { + case "70": + _, _ = w.Write([]byte(`{"ai_runtime_task_output": {"mlflow_experiment_id": "E1", "mlflow_run_id": "R0"}}`)) + case "71": + _, _ = w.Write([]byte(`{"ai_runtime_task_output": {"mlflow_experiment_id": "E1", "mlflow_run_id": "R1"}}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + case "/api/2.0/mlflow/runs/get": + _, _ = w.Write([]byte(`{"run": {"info": {"run_name": "train-` + r.URL.Query().Get("run_id") + `"}}}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + ctx := cmdio.MockDiscard(t.Context()) + w := newTestWorkspaceClient(t, srv.URL) + wantURL := srv.URL + "/ml/experiments/E1/runs/R1/logs" + + // When the list, the info pane and the logs pane render the run. + rows, err := newRunFetcher(ctx, w, listQuery{ + userFilter: "me@example.com", currentUser: "me@example.com", limit: 10, fetchMLflow: true, + }).next(10) + require.NoError(t, err) + info, err := runDetailText(ctx, w, 7) + require.NoError(t, err) + logs, err := runLogsSnapshot(ctx, w, 7) + require.NoError(t, err) + + // Then each links retry 1's MLflow run and names that retry. + require.Len(t, rows, 1) + assert.Equal(t, wantURL, rows[0].MLflowURL) + assert.Equal(t, "train-R1 (retry 1)", rows[0].MLflowLabel) + assert.Contains(t, info, "train-R1 (retry 1)") + assert.Contains(t, info, "MLflow URL: "+srv.URL+"/ml/experiments/E1/runs/R1\n") + assert.Contains(t, logs, "\nMLflow URL (retry 1): "+wantURL+"\n") +} diff --git a/cmd/air/list_index.go b/cmd/air/list_index.go index 52a70173a97..3bd1839f7b6 100644 --- a/cmd/air/list_index.go +++ b/cmd/air/list_index.go @@ -106,9 +106,9 @@ func (s *indexStrategy) hydrate(ids []int64) ([]listedRun, error) { // The cache key excludes the filter, so a cached row is still run through the // active filter; a non-matching hit is dropped, not re-fetched. for _, id := range ids { - if row, fields, taskRunID, ok := cachedRow(s.ctx, s.cache, host, id); ok { + if row, fields, tasks, ok := cachedRow(s.ctx, s.cache, host, id); ok { if s.filters.matchesFields(fields) { - rows = append(rows, listedRun{row: row, runID: id, taskRunID: taskRunID}) + rows = append(rows, listedRun{row: row, runID: id, tasks: tasks, terminal: true}) } continue } @@ -125,11 +125,11 @@ func (s *indexStrategy) hydrate(ids []int64) ([]listedRun, error) { continue } row := buildListRow(run, s.host, s.workspaceID) - taskRunID := taskRunID(run) - rows = append(rows, listedRun{row: row, runID: run.RunId, taskRunID: taskRunID}) + tasks := retryTasks(run) + rows = append(rows, listedRun{row: row, runID: run.RunId, tasks: tasks, terminal: isTerminal(run)}) if isTerminal(run) { start, _ := jobTiming(run) - putRow(s.ctx, s.cache, host, run.RunId, taskRunID, start, row, fields) + putRow(s.ctx, s.cache, host, run.RunId, tasks, start, row, fields) } } diff --git a/cmd/air/list_tui.go b/cmd/air/list_tui.go index 10ce1ca76dc..af5951bca44 100644 --- a/cmd/air/list_tui.go +++ b/cmd/air/list_tui.go @@ -9,6 +9,7 @@ import ( "github.com/charmbracelet/bubbles/viewport" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" + "github.com/charmbracelet/x/ansi" "github.com/databricks/cli/libs/browser" "github.com/databricks/cli/libs/cmdio" "github.com/spf13/cobra" @@ -122,7 +123,12 @@ type listModel struct { detailContent string // rendered detail content } +// detailHorizontalStep is how many columns ←/→ pan the detail pane. +const detailHorizontalStep = 8 + func newListModel(r *lipgloss.Renderer, f *runFetcher, rows []listRow, links bool) listModel { + vp := viewport.New(0, 0) + vp.SetHorizontalStep(detailHorizontalStep) return listModel{ rows: rows, styles: newListStyles(r), @@ -130,7 +136,7 @@ func newListModel(r *lipgloss.Renderer, f *runFetcher, rows []listRow, links boo links: links, fetcher: f, mode: modeList, - viewport: viewport.New(0, 0), + viewport: vp, } } @@ -184,6 +190,7 @@ func (m listModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.height = msg.Height m.viewport.Width = msg.Width m.viewport.Height = max(msg.Height-3, 1) + m.viewport.SetContent(m.wrappedDetail()) m.offset = m.clampedOffset() return m.maybeFetch() @@ -217,8 +224,9 @@ func (m listModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.detailContent = msg.body } m.detailTitle = msg.title - m.viewport.SetContent(m.detailContent) + m.viewport.SetContent(m.wrappedDetail()) m.viewport.GotoTop() + m.viewport.SetXOffset(0) return m, nil case tea.KeyMsg: @@ -292,6 +300,22 @@ func (m listModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m, nil } +// wrappedDetail is the detail text with long lines wrapped to the pane width, +// since the viewport clips them at the pane edge. Box rows and URLs stay on one +// row, because wrapping would break the box borders or the link; ←/→ pans to +// their clipped ends. +func (m listModel) wrappedDetail() string { + lines := strings.Split(m.detailContent, "\n") + for i, line := range lines { + plain := strings.TrimSpace(ansi.Strip(line)) + if strings.IndexAny(plain, "╭│╰") == 0 || strings.Contains(plain, "://") { + continue + } + lines[i] = ansi.Wrap(line, m.viewport.Width, "") + } + return strings.Join(lines, "\n") +} + // clampedOffset returns the scroll offset that keeps the cursor visible. func (m listModel) clampedOffset() int { visible := m.visibleCount() @@ -311,7 +335,7 @@ func (m listModel) View() string { } else { lines = append(lines, m.viewport.View()) } - footer := faint.Render("↑/↓ scroll · esc back · q quit") + footer := faint.Render("↑/↓ scroll · ←/→ pan · esc back · q quit") lines = append(lines, footer) return strings.Join(lines, "\n") } diff --git a/cmd/air/list_tui_test.go b/cmd/air/list_tui_test.go index 975371b7e8d..f195006f222 100644 --- a/cmd/air/list_tui_test.go +++ b/cmd/air/list_tui_test.go @@ -3,9 +3,11 @@ package aircmd import ( "io" "strconv" + "strings" "testing" tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/x/ansi" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/stretchr/testify/assert" @@ -285,6 +287,42 @@ func TestListModelDetailPaneAndBack(t *testing.T) { assert.Contains(t, m.View(), "Run ID") } +func TestListModelDetailPaneWrapsLongLines(t *testing.T) { + m := detailLoadingModel(t) + reason := "Failure reason: " + strings.Repeat("the task exited with a long error ", 4) + "ENDOFREASON" + require.Greater(t, len(reason), 80) + + next, _ := m.Update(detailMsg{title: "Run Details", body: reason}) + m = next.(listModel) + assert.Contains(t, m.View(), "ENDOFREASON", "the end of a long line must be visible, not clipped at the pane edge") + + // A narrower terminal re-wraps the same text. + next, _ = m.Update(tea.WindowSizeMsg{Width: 30, Height: 24}) + m = next.(listModel) + pane := m.viewport.View() + assert.Contains(t, pane, "ENDOFREASON") + for line := range strings.SplitSeq(pane, "\n") { + assert.LessOrEqual(t, ansi.StringWidth(line), 30, line) + } +} + +func TestListModelDetailPaneKeepsBoxRowsAndURLsWhole(t *testing.T) { + m := detailLoadingModel(t) + boxRow := "│ command: python train.py " + strings.Repeat("--flag ", 12) + "│" + url := "MLflow URL: https://example.test/ml/experiments/1/runs/" + strings.Repeat("a", 64) + require.Greater(t, len(boxRow), 80) + require.Greater(t, len(url), 80) + + next, _ := m.Update(detailMsg{title: "Run Details", body: boxRow + "\n" + url}) + m = next.(listModel) + assert.Equal(t, 2, m.viewport.TotalLineCount(), "box rows and URLs must not wrap") + + // ←/→ pans to the clipped end instead. + next, _ = m.Update(tea.KeyMsg{Type: tea.KeyRight}) + m = next.(listModel) + assert.Positive(t, m.viewport.HorizontalScrollPercent()) +} + func TestListModelDetailLateMsgDropped(t *testing.T) { m := detailLoadingModel(t) diff --git a/cmd/air/logstream_support.go b/cmd/air/logstream_support.go index 69896f71c3c..cb7a8d2d6d3 100644 --- a/cmd/air/logstream_support.go +++ b/cmd/air/logstream_support.go @@ -234,6 +234,14 @@ func emitNoLogs(out io.Writer, req logRequest, status logRunStatus) { if status.stateMessage != "" { msg = fmt.Sprintf("%s: %s", msg, status.stateMessage) } + // Only the latest attempt was checked; earlier ones may have logs. + if req.attempt < 0 && status.terminal() && status.latestAttempt > 0 { + retry := "0" + if status.latestAttempt > 1 { + retry = fmt.Sprintf("N (0 through %d)", status.latestAttempt-1) + } + msg = fmt.Sprintf("%s. Earlier retries may have logs; pass --retry %s to see them", strings.TrimRight(msg, "."), retry) + } if req.jsonOutput { printLogEvent(out, "ERROR", req.node, msg) return diff --git a/cmd/air/logstream_test.go b/cmd/air/logstream_test.go index fe0756d3862..b1006d9b49c 100644 --- a/cmd/air/logstream_test.go +++ b/cmd/air/logstream_test.go @@ -530,6 +530,7 @@ func TestMatchFatalPattern(t *testing.T) { func TestEmitNoLogs(t *testing.T) { tests := []struct { name string + req logRequest status logRunStatus want string }{ @@ -543,15 +544,42 @@ func TestEmitNoLogs(t *testing.T) { status: logRunStatus{lifeCycleState: "RUNNING"}, want: "No logs available yet for run 7, which is still in state RUNNING\n", }, + { + name: "latest retry of a terminal run", + req: logRequest{attempt: -1}, + status: logRunStatus{lifeCycleState: "TERMINATED", resultState: "FAILED", stateMessage: "Out of capacity.", latestAttempt: 2}, + want: "No logs available for run 7. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs; pass --retry N (0 through 1) to see them\n", + }, + { + name: "latest retry of a terminal run with one earlier retry", + req: logRequest{attempt: -1}, + status: logRunStatus{lifeCycleState: "TERMINATED", resultState: "FAILED", latestAttempt: 1}, + want: "No logs available for run 7. Run terminated in state FAILED. Earlier retries may have logs; pass --retry 0 to see them\n", + }, + { + name: "latest retry of a running run", + req: logRequest{attempt: -1}, + status: logRunStatus{lifeCycleState: "RUNNING", latestAttempt: 2}, + want: "No logs available yet for run 7, which is still in state RUNNING\n", + }, + { + name: "explicitly requested retry", + req: logRequest{attempt: 2}, + status: logRunStatus{lifeCycleState: "TERMINATED", resultState: "FAILED", latestAttempt: 2}, + want: "No logs available for run 7. Run terminated in state FAILED\n", + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var text bytes.Buffer - emitNoLogs(&text, logRequest{runID: 7}, tc.status) + req := tc.req + req.runID = 7 + emitNoLogs(&text, req, tc.status) assert.Equal(t, tc.want, text.String()) var jsonBuf bytes.Buffer - emitNoLogs(&jsonBuf, logRequest{runID: 7, node: 1, jsonOutput: true}, tc.status) + req.node, req.jsonOutput = 1, true + emitNoLogs(&jsonBuf, req, tc.status) var ev logEvent require.NoError(t, json.Unmarshal(jsonBuf.Bytes(), &ev)) assert.Equal(t, "ERROR", ev.Type) diff --git a/cmd/air/mlflow.go b/cmd/air/mlflow.go index 3e00ebf7492..7cf9f867630 100644 --- a/cmd/air/mlflow.go +++ b/cmd/air/mlflow.go @@ -3,6 +3,7 @@ package aircmd import ( "context" "fmt" + "slices" "strings" "github.com/databricks/cli/libs/log" @@ -15,19 +16,77 @@ import ( type mlflowIdentifiers struct { ExperimentID string RunID string + // OlderRetry is the retry the IDs came from when it isn't the run's latest + // (see mlflowIDsFromRetries), else nil. + OlderRetry *int } -// mlflowIDs fetches the MLflow IDs for a run via its latest task. Returns nil if -// they can't be obtained. +// retryNote is " (retry N)" when ids come from an older retry, else "". +func (ids *mlflowIdentifiers) retryNote() string { + if ids.OlderRetry == nil { + return "" + } + return fmt.Sprintf(" (retry %d)", *ids.OlderRetry) +} + +// mlflowIDs fetches the MLflow IDs a run's views link to (see +// mlflowIDsFromRetries). Returns nil if they can't be obtained. func mlflowIDs(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run) *mlflowIdentifiers { - return mlflowIDsFromOutput(aiRuntimeTaskOutput(ctx, w, run)) + return mlflowIDsForRetries(ctx, w, run.Tasks, isTerminal(run)) } -func aiRuntimeTaskOutput(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run) *jobs.AiRuntimeTaskOutput { - if len(run.Tasks) == 0 { +// mlflowIDsForRetries is mlflowIDs given only the task runs and whether the run +// has finished, which is what a cached list row keeps. +func mlflowIDsForRetries(ctx context.Context, w *databricks.WorkspaceClient, tasks []jobs.RunTask, terminal bool) *mlflowIdentifiers { + latest, err := latestTaskOutput(ctx, w, tasks) + return mlflowIDsFromRetries(ctx, w, tasks, terminal, latest, err) +} + +// mlflowIDsFromRetries returns the IDs from latest (the latest retry's output), +// else, for a finished run, from the newest earlier retry of the same task that +// has them: a last retry that failed before it started never got an MLflow run +// of its own. A run still in progress keeps only the latest retry's IDs, so its +// link and ETA never point at a previous retry. +// +// If a get-output call fails (latestErr, or an earlier retry's), it returns nil +// rather than skip that retry: the retry it couldn't read may have the newest +// MLflow run, and an older link would then look current. +func mlflowIDsFromRetries(ctx context.Context, w *databricks.WorkspaceClient, tasks []jobs.RunTask, terminal bool, latest *jobs.AiRuntimeTaskOutput, latestErr error) *mlflowIdentifiers { + if latestErr != nil { return nil } - return aiRuntimeTaskOutputForTask(ctx, w, run.Tasks[len(run.Tasks)-1].RunId) + if ids := mlflowIDsFromOutput(latest); ids != nil || len(tasks) < 2 || !terminal { + return ids + } + last := tasks[len(tasks)-1] + for _, task := range slices.Backward(tasks[:len(tasks)-1]) { + if task.TaskKey != last.TaskKey { + continue + } + out, err := aiRuntimeTaskOutputForTask(ctx, w, task.RunId) + if err != nil { + return nil + } + if ids := mlflowIDsFromOutput(out); ids != nil { + ids.OlderRetry = &task.AttemptNumber + return ids + } + } + return nil +} + +func aiRuntimeTaskOutput(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run) *jobs.AiRuntimeTaskOutput { + out, _ := latestTaskOutput(ctx, w, run.Tasks) + return out +} + +// latestTaskOutput fetches the output of the last of tasks, the run's latest +// retry (see aiRuntimeTaskOutputForTask). +func latestTaskOutput(ctx context.Context, w *databricks.WorkspaceClient, tasks []jobs.RunTask) (*jobs.AiRuntimeTaskOutput, error) { + if len(tasks) == 0 { + return nil, nil + } + return aiRuntimeTaskOutputForTask(ctx, w, tasks[len(tasks)-1].RunId) } // mlflowIDsForTask fetches a task run's MLflow experiment and run IDs from @@ -35,21 +94,24 @@ func aiRuntimeTaskOutput(ctx context.Context, w *databricks.WorkspaceClient, run // link, so any failure (endpoint error, run not yet started, no MLflow output) // is logged and treated as "no link" rather than failing the command. func mlflowIDsForTask(ctx context.Context, w *databricks.WorkspaceClient, taskRunID int64) *mlflowIdentifiers { - return mlflowIDsFromOutput(aiRuntimeTaskOutputForTask(ctx, w, taskRunID)) + out, _ := aiRuntimeTaskOutputForTask(ctx, w, taskRunID) + return mlflowIDsFromOutput(out) } -func aiRuntimeTaskOutputForTask(ctx context.Context, w *databricks.WorkspaceClient, taskRunID int64) *jobs.AiRuntimeTaskOutput { +// aiRuntimeTaskOutputForTask fetches a task run's AI runtime output, logging +// any fetch error before returning it. +func aiRuntimeTaskOutputForTask(ctx context.Context, w *databricks.WorkspaceClient, taskRunID int64) (*jobs.AiRuntimeTaskOutput, error) { if taskRunID == 0 { - return nil + return nil, nil } out, err := w.Jobs.GetRunOutputByRunId(ctx, taskRunID) if err != nil { log.Debugf(ctx, "air: could not fetch run output for MLflow link: %v", err) - return nil + return nil, err } - return out.AiRuntimeTaskOutput + return out.AiRuntimeTaskOutput, nil } func mlflowIDsFromOutput(output *jobs.AiRuntimeTaskOutput) *mlflowIdentifiers { diff --git a/cmd/air/mlflow_test.go b/cmd/air/mlflow_test.go index b72f7a928b3..2f316acaf74 100644 --- a/cmd/air/mlflow_test.go +++ b/cmd/air/mlflow_test.go @@ -63,18 +63,78 @@ func TestMLflowIDs(t *testing.T) { t.Run("uses the latest attempt's task run", func(t *testing.T) { // A retried run must link to the last task, not the stale first attempt. - var gotRunID string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/api/2.2/jobs/runs/get-output" { - gotRunID = r.URL.Query().Get("run_id") + runID := r.URL.Query().Get("run_id") + _, _ = w.Write([]byte(`{"ai_runtime_task_output":{"mlflow_experiment_id":"E1","mlflow_run_id":"R` + runID + `"}}`)) + return } _, _ = w.Write([]byte(`{}`)) })) t.Cleanup(srv.Close) retried := &jobs.Run{Tasks: []jobs.RunTask{{RunId: 99}, {RunId: 100}}} - mlflowIDs(ctx, newTestWorkspaceClient(t, srv.URL), retried) - assert.Equal(t, "100", gotRunID) + got := mlflowIDs(ctx, newTestWorkspaceClient(t, srv.URL), retried) + assert.Equal(t, &mlflowIdentifiers{ExperimentID: "E1", RunID: "R100"}, got) + }) + + t.Run("a finished run falls back to an earlier attempt when the latest has none", func(t *testing.T) { + // The last retry failed before it started, so only attempt 0 logged to MLflow. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/2.2/jobs/runs/get-output" && r.URL.Query().Get("run_id") == "99" { + _, _ = w.Write([]byte(`{"ai_runtime_task_output":{"mlflow_experiment_id":"E1","mlflow_run_id":"R1"}}`)) + return + } + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + + w := newTestWorkspaceClient(t, srv.URL) + tasks := []jobs.RunTask{{RunId: 99}, {RunId: 100, AttemptNumber: 1}} + finished := &jobs.Run{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated}, Tasks: tasks} + retry0 := 0 + assert.Equal(t, &mlflowIdentifiers{ExperimentID: "E1", RunID: "R1", OlderRetry: &retry0}, mlflowIDs(ctx, w, finished)) + + // A run still starting its next retry must not link to the previous attempt. + running := &jobs.Run{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}, Tasks: tasks} + assert.Nil(t, mlflowIDs(ctx, w, running)) + + // In a multi-task job, an earlier task is not a retry of the last one. + multiTask := &jobs.Run{ + State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated}, + Tasks: []jobs.RunTask{{TaskKey: "train", RunId: 99}, {TaskKey: "report", RunId: 100}}, + } + assert.Nil(t, mlflowIDs(ctx, w, multiTask)) + }) + + t.Run("a finished run links no older retry when a newer one's output can't be read", func(t *testing.T) { + // Given retry 0 logged to MLflow, but get-output fails for failingRunID. + finished := &jobs.Run{ + State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated}, + Tasks: []jobs.RunTask{{RunId: 98}, {RunId: 99, AttemptNumber: 1}, {RunId: 100, AttemptNumber: 2}}, + } + for _, failingRunID := range []string{"100", "99"} { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path != "/api/2.2/jobs/runs/get-output": + _, _ = w.Write([]byte(`{}`)) + case r.URL.Query().Get("run_id") == failingRunID: + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error_code":"PERMISSION_DENIED","message":"denied"}`)) + case r.URL.Query().Get("run_id") == "98": + _, _ = w.Write([]byte(`{"ai_runtime_task_output":{"mlflow_experiment_id":"E1","mlflow_run_id":"R0"}}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + + // When the views pick the run's MLflow run. + got := mlflowIDs(ctx, newTestWorkspaceClient(t, srv.URL), finished) + + // Then there is no link: the unread retry may have the newest MLflow run. + assert.Nil(t, got, "get-output fails for task run %s", failingRunID) + } }) } diff --git a/cmd/air/render.go b/cmd/air/render.go index f7a96f27b26..3985ab57588 100644 --- a/cmd/air/render.go +++ b/cmd/air/render.go @@ -124,7 +124,7 @@ func renderRunText(ctx context.Context, out io.Writer, w *databricks.WorkspaceCl } if ids != nil { - view.mlflowLabel = mlflowRunLabel(fetchMLflowRunName(ctx, w, ids.RunID), ids.RunID) + view.mlflowLabel = mlflowRunLabel(fetchMLflowRunName(ctx, w, ids.RunID), ids.RunID) + ids.retryNote() view.mlflowURL = mlflowRunURL(w.Config.Host, ids) }