Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/7456.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Wrap long failure reasons and log lines in the `air list` info and logs panes, keep the MLflow link (labeled with its retry) when a run's last retry failed before starting, and point `air logs` at earlier retries when the last one has no logs. ([#7456](https://github.com/databricks/cli/pull/7456))
1 change: 1 addition & 0 deletions acceptance/air/logs-failed-retry/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
7 changes: 7 additions & 0 deletions acceptance/air/logs-failed-retry/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@

=== logs of a failed retry with no logs point to earlier retries

>>> [CLI] air logs 123
No logs available for run 123. Run terminated in state FAILED: Out of capacity. Earlier retries may have logs; pass --retry N (0 through 1) to see them

Exit code: 1
2 changes: 2 additions & 0 deletions acceptance/air/logs-failed-retry/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
title "logs of a failed retry with no logs point to earlier retries"
errcode trace $CLI air logs 123
31 changes: 31 additions & 0 deletions acceptance/air/logs-failed-retry/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
# The SDK occasionally probes host reachability with a HEAD request; stub it so
# the test is deterministic.
[[Server]]
Pattern = "HEAD /"
Response.Body = ''

# A failed run whose second and third attempts failed before they started.
[[Server]]
Pattern = "GET /api/2.2/jobs/runs/get"
Response.Body = '''
{
"run_id": 123,
"start_time": 1700000000000,
"end_time": 1700000012000,
"state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "Out of capacity"},
"tasks": [
{"task_key": "train", "run_id": 456, "attempt_number": 0},
{"task_key": "train", "run_id": 457, "attempt_number": 1},
{"task_key": "train", "run_id": 458, "attempt_number": 2}
]
}
'''

# Neither Bricklens nor MLflow has logs for the latest attempt.
[[Server]]
Pattern = "GET /api/2.0/ai-training/workflows/by-run-id/123/logs"
Response.Body = '{"log_records": []}'

[[Server]]
Pattern = "GET /api/2.2/jobs/runs/get-output"
Response.Body = '{}'
4 changes: 2 additions & 2 deletions cmd/air/get.go
Original file line number Diff line number Diff line change
Expand Up @@ -167,8 +167,8 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`,

data := buildGetData(run)
data.DashboardURL = dashboardURL(w.Config.Host, runID, workspaceID)
taskOutput := aiRuntimeTaskOutput(ctx, w, run)
ids := mlflowIDsFromOutput(taskOutput)
taskOutput, taskOutputErr := latestTaskOutput(ctx, w, run.Tasks)
ids := mlflowIDsFromRetries(ctx, w, run.Tasks, isTerminal(run), taskOutput, taskOutputErr)
if ids != nil {
url := mlflowLogsURL(w.Config.Host, ids)
data.MLflowURL = &url
Expand Down
12 changes: 7 additions & 5 deletions cmd/air/joblist.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,14 @@ func isSweep(r *jobs.Run) bool {
return len(r.Tasks) > 0 && r.Tasks[0].ForEachTask != nil
}

// taskRunID returns the run id of the AIR task, used to fetch its MLflow output.
func taskRunID(r *jobs.Run) int64 {
if len(r.Tasks) == 0 {
return 0
// retryTasks returns the run's task runs with only the fields that pick its
// MLflow run (see mlflowIDsFromRetries), so the list cache stays small.
func retryTasks(r *jobs.Run) []jobs.RunTask {
tasks := make([]jobs.RunTask, len(r.Tasks))
for i, t := range r.Tasks {
tasks[i] = jobs.RunTask{RunId: t.RunId, TaskKey: t.TaskKey, AttemptNumber: t.AttemptNumber}
}
return r.Tasks[0].RunId
return tasks
}

// jobExperiment returns the run's MLflow experiment name (user-folder prefix
Expand Down
21 changes: 12 additions & 9 deletions cmd/air/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,13 @@ type listRow struct {
Accelerators string `json:"-"`
}

// listedRun pairs a row with its parent and task run ids, so the MLflow link can
// be fetched after the run has been filtered in and cached afterward.
// listedRun pairs a row with its parent run id and task runs, so the MLflow link
// can be fetched after the run has been filtered in and cached afterward.
type listedRun struct {
row listRow
runID int64
taskRunID int64
row listRow
runID int64
tasks []jobs.RunTask // see retryTasks
terminal bool
}

// listQuery holds the resolved inputs to a runFetcher.
Expand Down Expand Up @@ -310,7 +311,7 @@ func (s *jobsScanStrategy) next(want int) ([]listedRun, error) {
if !s.filters.matches(run) {
continue
}
entries = append(entries, listedRun{row: buildListRow(run, s.host, s.workspaceID), runID: run.RunId, taskRunID: taskRunID(run)})
entries = append(entries, listedRun{row: buildListRow(run, s.host, s.workspaceID), runID: run.RunId, tasks: retryTasks(run), terminal: isTerminal(run)})
}
return entries, nil
}
Expand Down Expand Up @@ -343,13 +344,15 @@ func setMLflowLinks(ctx context.Context, w *databricks.WorkspaceClient, host str
if entries[i].row.MLflowURL != "" && entries[i].row.MLflowURL != "-" {
return nil
}
ids := mlflowIDsForTask(ctx, w, entries[i].taskRunID)
ids := mlflowIDsForRetries(ctx, w, entries[i].tasks, entries[i].terminal)
if ids == nil {
return nil
}
entries[i].row.MLflowURL = mlflowLogsURL(host, ids)
name := fetchMLflowRunName(ctx, w, ids.RunID)
entries[i].row.MLflowLabel = mlflowRunLabel(name, ids.RunID)
// Shorten the name, not the retry note, to fit the column.
note := ids.retryNote()
entries[i].row.MLflowLabel = truncate(mlflowRunLabel(name, ids.RunID), mlflowColWidth-len(note)) + note
entries[i].row.ExperimentURL = mlflowExperimentURL(host, ids)
// The ETA needs a progress-metric history fetch, so it's computed only
// for running rows (the only ones that can have one).
Expand All @@ -361,6 +364,6 @@ func setMLflowLinks(ctx context.Context, w *databricks.WorkspaceClient, host str
return nil
})
}
// mlflowIDsForTask never returns an error (it logs and yields nil), so Wait can't fail.
// mlflowIDsForRetries never returns an error (it logs and yields nil), so Wait can't fail.
_ = g.Wait()
}
27 changes: 15 additions & 12 deletions cmd/air/list_cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"time"

"github.com/databricks/cli/libs/cache"
"github.com/databricks/databricks-sdk-go/service/jobs"
)

// The AiTrainingService index path caches hydrated terminal runs on disk:
Expand Down Expand Up @@ -41,9 +42,11 @@ type cachedRun struct {
RunURL string `json:"run_url"`
ExperimentURL string `json:"experiment_url"`
Accelerators string `json:"accelerators"`
TaskRunID int64 `json:"task_run_id"`
Fields filterFields `json:"filter_fields"`
SubmitTimeMs int64 `json:"submit_time_ms"`

// Tasks are the run's task runs (see retryTasks), which pick its MLflow link.
Tasks []jobs.RunTask `json:"tasks"`
}

func (c cachedRun) toRow() listRow {
Expand All @@ -55,13 +58,13 @@ func (c cachedRun) toRow() listRow {
}
}

func cachedRunFromRow(r listRow, taskRunID int64, fields filterFields, submitTimeMs int64) cachedRun {
func cachedRunFromRow(r listRow, tasks []jobs.RunTask, fields filterFields, submitTimeMs int64) cachedRun {
return cachedRun{
RunID: r.RunID, RunName: r.RunName, User: r.User, Status: r.Status,
StartedAt: r.StartedAt, IsSweep: r.IsSweep, Experiment: r.Experiment,
Duration: r.Duration, MLflowURL: r.MLflowURL, MLflowLabel: r.MLflowLabel,
RunURL: r.RunURL, ExperimentURL: r.ExperimentURL, Accelerators: r.Accelerators,
TaskRunID: taskRunID, Fields: fields, SubmitTimeMs: submitTimeMs,
Tasks: tasks, Fields: fields, SubmitTimeMs: submitTimeMs,
}
}

Expand All @@ -71,19 +74,19 @@ func newListCache(ctx context.Context) *cache.Cache {
return cache.NewCache(ctx, listCacheComponent, listCacheTTL, nil)
}

// cachedRow returns the cached row, filter fields, and task run id for a run.
// Entries written before task ids were cached are treated as misses.
func cachedRow(ctx context.Context, c *cache.Cache, host string, runID int64) (listRow, filterFields, int64, bool) {
// cachedRow returns the cached row, filter fields, and task runs for a run.
// Entries written before task runs were cached are treated as misses.
func cachedRow(ctx context.Context, c *cache.Cache, host string, runID int64) (listRow, filterFields, []jobs.RunTask, bool) {
entry, ok := cache.Get[cachedRun](ctx, c, listCacheKey{Host: host, RunID: runID})
if !ok || entry.TaskRunID == 0 {
return listRow{}, filterFields{}, 0, false
if !ok || len(entry.Tasks) == 0 {
return listRow{}, filterFields{}, nil, false
}
return entry.toRow(), entry.Fields, entry.TaskRunID, true
return entry.toRow(), entry.Fields, entry.Tasks, true
}

// putRow caches a terminal run's base row and filter fields under its submit time.
func putRow(ctx context.Context, c *cache.Cache, host string, runID, taskRunID, submitTimeMs int64, row listRow, fields filterFields) {
cache.Put(ctx, c, listCacheKey{Host: host, RunID: runID}, cachedRunFromRow(row, taskRunID, fields, submitTimeMs))
func putRow(ctx context.Context, c *cache.Cache, host string, runID int64, tasks []jobs.RunTask, submitTimeMs int64, row listRow, fields filterFields) {
cache.Put(ctx, c, listCacheKey{Host: host, RunID: runID}, cachedRunFromRow(row, tasks, fields, submitTimeMs))
}

// updateCachedRow replaces the display row while preserving cache metadata.
Expand All @@ -93,5 +96,5 @@ func updateCachedRow(ctx context.Context, c *cache.Cache, host string, runID int
if !ok {
return
}
cache.Put(ctx, c, key, cachedRunFromRow(row, entry.TaskRunID, entry.Fields, entry.SubmitTimeMs))
cache.Put(ctx, c, key, cachedRunFromRow(row, entry.Tasks, entry.Fields, entry.SubmitTimeMs))
}
21 changes: 13 additions & 8 deletions cmd/air/list_cache_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"net/http/httptest"
"testing"

"github.com/databricks/cli/libs/cache"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand All @@ -21,20 +22,24 @@ func TestListCacheRoundTrip(t *testing.T) {

row := listRow{RunID: "42", Experiment: "exp", Status: "SUCCESS"}
fields := filterFields{Experiment: "exp", GPUType: "GPU_1xA10", GPUCount: 1}
putRow(ctx, c, "https://host.test", 42, 420, 1700000000000, row, fields)
tasks := []jobs.RunTask{{RunId: 420, TaskKey: "train"}, {RunId: 421, TaskKey: "train", AttemptNumber: 1}}
putRow(ctx, c, "https://host.test", 42, tasks, 1700000000000, row, fields)

got, gotFields, taskRunID, ok := cachedRow(ctx, c, "https://host.test", 42)
got, gotFields, gotTasks, ok := cachedRow(ctx, c, "https://host.test", 42)
require.True(t, ok, "hit after write")
assert.Equal(t, row, got)
assert.Equal(t, fields, gotFields)
assert.Equal(t, int64(420), taskRunID)
for i := range gotTasks {
gotTasks[i].ForceSendFields = nil // set by the JSON round trip
}
assert.Equal(t, tasks, gotTasks)

// Different host is a different key.
_, _, _, ok = cachedRow(ctx, c, "https://other.test", 42)
assert.False(t, ok)

// Entries from the previous cache format have no task run id and must be refreshed.
putRow(ctx, c, "https://host.test", 43, 0, 1700000000000, listRow{RunID: "43"}, filterFields{})
// Entries from previous cache formats have no task runs and must be refreshed.
cache.Put(ctx, c, listCacheKey{Host: "https://host.test", RunID: 43}, map[string]any{"run_id": "43", "task_run_id": 430})
_, _, _, ok = cachedRow(ctx, c, "https://host.test", 43)
assert.False(t, ok)
}
Expand All @@ -48,7 +53,7 @@ func TestIndexStrategyServesCachedRowWithoutFetch(t *testing.T) {

// Pre-seed the cache for run 7 so hydration should skip runs/get entirely.
ctx := t.Context()
putRow(ctx, newListCache(ctx), host, 7, 70, 1000_000, listRow{RunID: "7", Status: "SUCCESS"}, filterFields{})
putRow(ctx, newListCache(ctx), host, 7, []jobs.RunTask{{RunId: 70}}, 1000_000, listRow{RunID: "7", Status: "SUCCESS"}, filterFields{})

f := newRunFetcher(ctx, newTestWorkspaceClient(t, host), listQuery{
userFilter: "me@example.com", currentUser: "me@example.com", limit: 10,
Expand All @@ -69,7 +74,7 @@ func TestIndexStrategyFiltersCachedRow(t *testing.T) {

// Cache hit under experiment "bar" must be filtered out by an experiment=foo query.
ctx := t.Context()
putRow(ctx, newListCache(ctx), host, 7, 70, 1000_000,
putRow(ctx, newListCache(ctx), host, 7, []jobs.RunTask{{RunId: 70}}, 1000_000,
listRow{RunID: "7", Status: "SUCCESS", Experiment: "bar"},
filterFields{Experiment: "bar"})

Expand All @@ -91,7 +96,7 @@ func TestIndexStrategyServesMatchingCachedRow(t *testing.T) {
host := srv.URL

ctx := t.Context()
putRow(ctx, newListCache(ctx), host, 7, 70, 1000_000,
putRow(ctx, newListCache(ctx), host, 7, []jobs.RunTask{{RunId: 70}}, 1000_000,
listRow{RunID: "7", Status: "SUCCESS", Experiment: "foo"},
filterFields{Experiment: "foo"})

Expand Down
8 changes: 6 additions & 2 deletions cmd/air/list_detail.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,16 @@ func runDetailText(ctx context.Context, w *databricks.WorkspaceClient, runID int
}

// runLogsSnapshot fetches a one-shot tail of a run's logs into a string, for the
// list picker's detail pane.
// list picker's detail pane, followed by the run's MLflow logs link.
func runLogsSnapshot(ctx context.Context, w *databricks.WorkspaceClient, runID int64) (string, error) {
status, err := resolveRunStatus(ctx, w, runID)
run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: runID})
if err != nil {
if errors.Is(err, apierr.ErrResourceDoesNotExist) {
return "", fmt.Errorf("run %d not found", runID)
}
return "", fmt.Errorf("failed to fetch run status: %w", err)
}
status := projectRunStatus(run)

req := logRequest{
runID: runID,
Expand All @@ -60,5 +61,8 @@ func runLogsSnapshot(ctx context.Context, w *databricks.WorkspaceClient, runID i
if _, err := fetchLogs(ctx, w, &buf, req, status); err != nil {
return "", fmt.Errorf("failed to fetch logs: %w", err)
}
if ids := mlflowIDs(ctx, w, run); ids != nil {
fmt.Fprintf(&buf, "\nMLflow URL%s: %s\n", ids.retryNote(), mlflowLogsURL(w.Config.Host, ids))
}
return buf.String(), nil
}
97 changes: 97 additions & 0 deletions cmd/air/list_detail_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
package aircmd

import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"

"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestRunLogsSnapshotFailedRetryWithoutLogs(t *testing.T) {
// Only attempt 0 ran: retries 1 and 2 failed before they started, with no
// logs or MLflow output of their own. The pane still links the MLflow run
// that attempt 0 logged to.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/logs"):
_, _ = w.Write([]byte(`{"log_records": []}`))
case r.URL.Path == "/api/2.2/jobs/runs/get":
_, _ = w.Write([]byte(`{"run_id": 123, "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED", "state_message": "Out of capacity"},
"tasks": [{"run_id": 456, "attempt_number": 0}, {"run_id": 457, "attempt_number": 1}, {"run_id": 458, "attempt_number": 2}]}`))
case r.URL.Path == "/api/2.2/jobs/runs/get-output" && r.URL.Query().Get("run_id") == "456":
_, _ = w.Write([]byte(`{"ai_runtime_task_output": {"mlflow_experiment_id": "E1", "mlflow_run_id": "R1"}}`))
default:
_, _ = w.Write([]byte(`{}`))
}
}))
t.Cleanup(srv.Close)

got, err := runLogsSnapshot(t.Context(), newTestWorkspaceClient(t, srv.URL), 123)
require.NoError(t, err)
assert.Equal(t, "No logs available for run 123. Run terminated in state FAILED: Out of capacity. "+
"Earlier retries may have logs; pass --retry N (0 through 1) to see them\n"+
"\nMLflow URL (retry 0): "+srv.URL+"/ml/experiments/E1/runs/R1/logs\n", got)
}

func TestListRowAndPanesLinkTheSameOlderRetry(t *testing.T) {
// Given a failed run whose retries 0 and 1 logged to MLflow and whose last
// retry failed before it started.
t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir())
run := indexRun(7, 1000_000)
run.State.ResultState = jobs.RunResultStateFailed
task := run.Tasks[0]
run.Tasks = nil
for attempt, taskRunID := range []int64{70, 71, 72} {
task.RunId, task.AttemptNumber = taskRunID, attempt
run.Tasks = append(run.Tasks, task)
}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case aiTrainingWorkflowsPath:
_, _ = w.Write([]byte(`{"training_workflows": [{"job_run_id": "7", "submit_time": {"seconds": 1000}}]}`))
case "/api/2.2/jobs/runs/get":
_ = json.NewEncoder(w).Encode(run)
case "/api/2.2/jobs/runs/get-output":
switch r.URL.Query().Get("run_id") {
case "70":
_, _ = w.Write([]byte(`{"ai_runtime_task_output": {"mlflow_experiment_id": "E1", "mlflow_run_id": "R0"}}`))
case "71":
_, _ = w.Write([]byte(`{"ai_runtime_task_output": {"mlflow_experiment_id": "E1", "mlflow_run_id": "R1"}}`))
default:
_, _ = w.Write([]byte(`{}`))
}
case "/api/2.0/mlflow/runs/get":
_, _ = w.Write([]byte(`{"run": {"info": {"run_name": "train-` + r.URL.Query().Get("run_id") + `"}}}`))
default:
_, _ = w.Write([]byte(`{}`))
}
}))
t.Cleanup(srv.Close)
ctx := cmdio.MockDiscard(t.Context())
w := newTestWorkspaceClient(t, srv.URL)
wantURL := srv.URL + "/ml/experiments/E1/runs/R1/logs"

// When the list, the info pane and the logs pane render the run.
rows, err := newRunFetcher(ctx, w, listQuery{
userFilter: "me@example.com", currentUser: "me@example.com", limit: 10, fetchMLflow: true,
}).next(10)
require.NoError(t, err)
info, err := runDetailText(ctx, w, 7)
require.NoError(t, err)
logs, err := runLogsSnapshot(ctx, w, 7)
require.NoError(t, err)

// Then each links retry 1's MLflow run and names that retry.
require.Len(t, rows, 1)
assert.Equal(t, wantURL, rows[0].MLflowURL)
assert.Equal(t, "train-R1 (retry 1)", rows[0].MLflowLabel)
assert.Contains(t, info, "train-R1 (retry 1)")
assert.Contains(t, info, "MLflow URL: "+srv.URL+"/ml/experiments/E1/runs/R1\n")
assert.Contains(t, logs, "\nMLflow URL (retry 1): "+wantURL+"\n")
}
Loading
Loading