diff --git a/.nextchanges/air/7627.md b/.nextchanges/air/7627.md new file mode 100644 index 00000000000..fc5d87baf2a --- /dev/null +++ b/.nextchanges/air/7627.md @@ -0,0 +1 @@ +* `air list --all-users` and `air list --filter user=` cache finished runs' MLflow links, so repeat listings are about 3x faster. ([#7627](https://github.com/databricks/cli/pull/7627)) diff --git a/cmd/air/list.go b/cmd/air/list.go index 1627470dc8a..8b874d517c3 100644 --- a/cmd/air/list.go +++ b/cmd/air/list.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cache" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/log" @@ -271,10 +272,17 @@ func (f *runFetcher) next(want int) ([]listRow, error) { // MLflow links appear only in the text table, so the per-run get-output // lookups are skipped for JSON output (which omits the column anyway). if f.fetchMLflow { + scan, isScan := f.strategy.(*jobsScanStrategy) + if isScan { + applyCachedMLflowLinks(f.ctx, scan.mlflowCache, f.w.Config.Host, entries) + } setMLflowLinks(f.ctx, f.w, f.w.Config.Host, entries) if s, ok := f.strategy.(*indexStrategy); ok { s.cacheMLflowLinks(entries) } + if isScan { + putMLflowLinks(f.ctx, scan.mlflowCache, f.w.Config.Host, entries) + } } rows := make([]listRow, len(entries)) @@ -296,6 +304,7 @@ type jobsScanStrategy struct { filters listFilters host string workspaceID int64 + mlflowCache *cache.Cache // terminal runs' MLflow links scanned int } @@ -319,6 +328,7 @@ func newJobsScanStrategy(ctx context.Context, w *databricks.WorkspaceClient, q l userFilter: q.userFilter, currentUser: q.currentUser, filters: q.filters, + mlflowCache: newScanMLflowCache(ctx), host: w.Config.Host, workspaceID: q.workspaceID, } diff --git a/cmd/air/list_cache.go b/cmd/air/list_cache.go index 941327e93b1..37e3fe84398 100644 --- a/cmd/air/list_cache.go +++ b/cmd/air/list_cache.go @@ -2,6 +2,7 @@ package aircmd import ( "context" + "slices" "time" "github.com/databricks/cli/libs/cache" @@ -96,6 +97,66 @@ func putRow(ctx context.Context, c *cache.Cache, host string, runID int64, tasks cache.Put(ctx, c, listCacheKey{Host: host, RunID: runID}, cachedRunFromRow(row, tasks, fields, submitTimeMs)) } +// The Jobs scan builds its rows from runs/list on every call, so it caches only +// the MLflow links of terminal runs, which cost a get-output and an MLflow +// request per row. It uses its own component so its entries never stand in for +// the index path's full rows. +const listScanMLflowCacheComponent = "air-list-scan-mlflow-v1" + +// cachedMLflowLink is a terminal run's MLflow link fields, with the task runs +// they were resolved from. +type cachedMLflowLink struct { + MLflowURL string `json:"mlflow_url"` + MLflowLabel string `json:"mlflow_label"` + ExperimentURL string `json:"experiment_url"` + Tasks []jobs.RunTask `json:"tasks"` +} + +func newScanMLflowCache(ctx context.Context) *cache.Cache { + return cache.NewCache(ctx, listScanMLflowCacheComponent, listCacheTTL, nil) +} + +// applyCachedMLflowLinks fills terminal entries' MLflow fields from the cache. +// An entry is used only while the run's task runs and attempts still match the +// ones it was resolved from, so a run repaired after it was cached is refetched. +func applyCachedMLflowLinks(ctx context.Context, c *cache.Cache, host string, entries []listedRun) { + for i := range entries { + if !entries[i].terminal { + continue + } + link, ok := cache.Get[cachedMLflowLink](ctx, c, listCacheKey{Host: host, RunID: entries[i].runID}) + if !ok || !sameTaskRuns(link.Tasks, entries[i].tasks) { + continue + } + entries[i].row.MLflowURL = link.MLflowURL + entries[i].row.MLflowLabel = link.MLflowLabel + entries[i].row.ExperimentURL = link.ExperimentURL + } +} + +// putMLflowLinks caches terminal entries that have an MLflow link. +func putMLflowLinks(ctx context.Context, c *cache.Cache, host string, entries []listedRun) { + for _, e := range entries { + if !e.terminal || e.row.MLflowURL == "" || e.row.MLflowURL == "-" { + continue + } + cache.Put(ctx, c, listCacheKey{Host: host, RunID: e.runID}, cachedMLflowLink{ + MLflowURL: e.row.MLflowURL, + MLflowLabel: e.row.MLflowLabel, + ExperimentURL: e.row.ExperimentURL, + Tasks: e.tasks, + }) + } +} + +// sameTaskRuns reports whether two retryTasks lists name the same task runs and +// attempts. +func sameTaskRuns(a, b []jobs.RunTask) bool { + return slices.EqualFunc(a, b, func(x, y jobs.RunTask) bool { + return x.RunId == y.RunId && x.TaskKey == y.TaskKey && x.AttemptNumber == y.AttemptNumber + }) +} + // updateCachedRow replaces the display row while preserving cache metadata. func updateCachedRow(ctx context.Context, c *cache.Cache, host string, runID int64, row listRow) { key := listCacheKey{Host: host, RunID: runID} diff --git a/cmd/air/list_cache_test.go b/cmd/air/list_cache_test.go index 00c821ac010..9e92f116cf6 100644 --- a/cmd/air/list_cache_test.go +++ b/cmd/air/list_cache_test.go @@ -171,6 +171,72 @@ func TestIndexStrategyCachesMLflowEnrichment(t *testing.T) { assert.Equal(t, []string{"70"}, outputRunIDs, "get-output must use the task run id") } +func TestJobsScanCachesTerminalMLflowLinks(t *testing.T) { + t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir()) + + // scanRun is a runs/list AIR run; one task run per attempt, newest last. + scanRun := func(id int64, state jobs.RunLifeCycleState, attempts ...int) jobs.BaseRun { + r := airBaseRun(id, "me@example.com", "GPU_1xA10", 1, "exp") + r.State = &jobs.RunState{LifeCycleState: state} + task := r.Tasks[0] + r.Tasks = nil + for _, a := range attempts { + task.RunId = id*10 + int64(a) + task.AttemptNumber = a + r.Tasks = append(r.Tasks, task) + } + return r + } + var runs []jobs.BaseRun + var outputHits, mlflowHits int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/2.2/jobs/runs/list": + _, _ = w.Write([]byte(runsListBody(t, "", runs...))) + case "/api/2.2/jobs/runs/get-output": + outputHits++ + _, _ = w.Write([]byte(`{"ai_runtime_task_output":{"mlflow_experiment_id":"exp1","mlflow_run_id":"m-` + r.URL.Query().Get("run_id") + `"}}`)) + case "/api/2.0/mlflow/runs/get": + mlflowHits++ + _, _ = w.Write([]byte(`{"run":{"info":{"run_name":"training-run"}}}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + list := func() []listRow { + rows, err := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{ + userFilter: "me@example.com", fetchMLflow: true, limit: 10, + }).next(10) + require.NoError(t, err) + require.Len(t, rows, 1) + return rows + } + + // A terminal run's links are resolved once, then served from the cache. + runs = []jobs.BaseRun{scanRun(7, jobs.RunLifeCycleStateTerminated, 0)} + first := list() + assert.Equal(t, srv.URL+"/ml/experiments/exp1/runs/m-70/logs", first[0].MLflowURL) + assert.Equal(t, first, list()) + assert.Equal(t, 1, outputHits) + assert.Equal(t, 1, mlflowHits) + + // A repaired run has a new attempt, so its cached links are stale. + runs = []jobs.BaseRun{scanRun(7, jobs.RunLifeCycleStateTerminated, 0, 1)} + assert.Equal(t, srv.URL+"/ml/experiments/exp1/runs/m-71/logs", list()[0].MLflowURL) + assert.Equal(t, 2, outputHits) + + // An active run is never served from the cache. + runs = []jobs.BaseRun{scanRun(8, jobs.RunLifeCycleStateRunning, 0)} + list() + list() + assert.Equal(t, 4, outputHits) + + // The scan's entries never stand in for the index path's full rows. + _, _, _, ok := cachedRow(t.Context(), newListCache(t.Context()), srv.URL, 7) + assert.False(t, ok) +} + func TestIsTerminal(t *testing.T) { assert.True(t, isTerminal(&jobs.Run{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated}})) assert.True(t, isTerminal(&jobs.Run{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateInternalError}}))