Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/7627.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `air list --all-users` and `air list --filter user=` cache finished runs' MLflow links, so repeat listings are about 3x faster. ([#7627](https://github.com/databricks/cli/pull/7627))
10 changes: 10 additions & 0 deletions cmd/air/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"sync"

"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/cache"
"github.com/databricks/cli/libs/cmdctx"
"github.com/databricks/cli/libs/flags"
"github.com/databricks/cli/libs/log"
Expand Down Expand Up @@ -271,10 +272,17 @@ func (f *runFetcher) next(want int) ([]listRow, error) {
// MLflow links appear only in the text table, so the per-run get-output
// lookups are skipped for JSON output (which omits the column anyway).
if f.fetchMLflow {
scan, isScan := f.strategy.(*jobsScanStrategy)
if isScan {
applyCachedMLflowLinks(f.ctx, scan.mlflowCache, f.w.Config.Host, entries)
}
setMLflowLinks(f.ctx, f.w, f.w.Config.Host, entries)
if s, ok := f.strategy.(*indexStrategy); ok {
s.cacheMLflowLinks(entries)
}
if isScan {
putMLflowLinks(f.ctx, scan.mlflowCache, f.w.Config.Host, entries)
}
}

rows := make([]listRow, len(entries))
Expand All @@ -296,6 +304,7 @@ type jobsScanStrategy struct {
filters listFilters
host string
workspaceID int64
mlflowCache *cache.Cache // terminal runs' MLflow links

scanned int
}
Expand All @@ -319,6 +328,7 @@ func newJobsScanStrategy(ctx context.Context, w *databricks.WorkspaceClient, q l
userFilter: q.userFilter,
currentUser: q.currentUser,
filters: q.filters,
mlflowCache: newScanMLflowCache(ctx),
host: w.Config.Host,
workspaceID: q.workspaceID,
}
Expand Down
61 changes: 61 additions & 0 deletions cmd/air/list_cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package aircmd

import (
"context"
"slices"
"time"

"github.com/databricks/cli/libs/cache"
Expand Down Expand Up @@ -96,6 +97,66 @@ func putRow(ctx context.Context, c *cache.Cache, host string, runID int64, tasks
cache.Put(ctx, c, listCacheKey{Host: host, RunID: runID}, cachedRunFromRow(row, tasks, fields, submitTimeMs))
}

// The Jobs scan builds its rows from runs/list on every call, so it caches only
// the MLflow links of terminal runs, which cost a get-output and an MLflow
// request per row. It uses its own component so its entries never stand in for
// the index path's full rows.
const listScanMLflowCacheComponent = "air-list-scan-mlflow-v1"

// cachedMLflowLink is a terminal run's MLflow link fields, with the task runs
// they were resolved from.
type cachedMLflowLink struct {
MLflowURL string `json:"mlflow_url"`
MLflowLabel string `json:"mlflow_label"`
ExperimentURL string `json:"experiment_url"`
Tasks []jobs.RunTask `json:"tasks"`
}

func newScanMLflowCache(ctx context.Context) *cache.Cache {
return cache.NewCache(ctx, listScanMLflowCacheComponent, listCacheTTL, nil)
}

// applyCachedMLflowLinks fills terminal entries' MLflow fields from the cache.
// An entry is used only while the run's task runs and attempts still match the
// ones it was resolved from, so a run repaired after it was cached is refetched.
func applyCachedMLflowLinks(ctx context.Context, c *cache.Cache, host string, entries []listedRun) {
for i := range entries {
if !entries[i].terminal {
continue
}
link, ok := cache.Get[cachedMLflowLink](ctx, c, listCacheKey{Host: host, RunID: entries[i].runID})
if !ok || !sameTaskRuns(link.Tasks, entries[i].tasks) {
continue
}
entries[i].row.MLflowURL = link.MLflowURL
entries[i].row.MLflowLabel = link.MLflowLabel
entries[i].row.ExperimentURL = link.ExperimentURL
}
}

// putMLflowLinks caches terminal entries that have an MLflow link.
func putMLflowLinks(ctx context.Context, c *cache.Cache, host string, entries []listedRun) {
for _, e := range entries {
if !e.terminal || e.row.MLflowURL == "" || e.row.MLflowURL == "-" {
continue
}
cache.Put(ctx, c, listCacheKey{Host: host, RunID: e.runID}, cachedMLflowLink{
MLflowURL: e.row.MLflowURL,
MLflowLabel: e.row.MLflowLabel,
ExperimentURL: e.row.ExperimentURL,
Tasks: e.tasks,
})
}
}

// sameTaskRuns reports whether two retryTasks lists name the same task runs and
// attempts.
func sameTaskRuns(a, b []jobs.RunTask) bool {
return slices.EqualFunc(a, b, func(x, y jobs.RunTask) bool {
return x.RunId == y.RunId && x.TaskKey == y.TaskKey && x.AttemptNumber == y.AttemptNumber
})
}

// updateCachedRow replaces the display row while preserving cache metadata.
func updateCachedRow(ctx context.Context, c *cache.Cache, host string, runID int64, row listRow) {
key := listCacheKey{Host: host, RunID: runID}
Expand Down
66 changes: 66 additions & 0 deletions cmd/air/list_cache_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,72 @@ func TestIndexStrategyCachesMLflowEnrichment(t *testing.T) {
assert.Equal(t, []string{"70"}, outputRunIDs, "get-output must use the task run id")
}

func TestJobsScanCachesTerminalMLflowLinks(t *testing.T) {
t.Setenv("DATABRICKS_CACHE_DIR", t.TempDir())

// scanRun is a runs/list AIR run; one task run per attempt, newest last.
scanRun := func(id int64, state jobs.RunLifeCycleState, attempts ...int) jobs.BaseRun {
r := airBaseRun(id, "me@example.com", "GPU_1xA10", 1, "exp")
r.State = &jobs.RunState{LifeCycleState: state}
task := r.Tasks[0]
r.Tasks = nil
for _, a := range attempts {
task.RunId = id*10 + int64(a)
task.AttemptNumber = a
r.Tasks = append(r.Tasks, task)
}
return r
}
var runs []jobs.BaseRun
var outputHits, mlflowHits int
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/2.2/jobs/runs/list":
_, _ = w.Write([]byte(runsListBody(t, "", runs...)))
case "/api/2.2/jobs/runs/get-output":
outputHits++
_, _ = w.Write([]byte(`{"ai_runtime_task_output":{"mlflow_experiment_id":"exp1","mlflow_run_id":"m-` + r.URL.Query().Get("run_id") + `"}}`))
case "/api/2.0/mlflow/runs/get":
mlflowHits++
_, _ = w.Write([]byte(`{"run":{"info":{"run_name":"training-run"}}}`))
default:
_, _ = w.Write([]byte(`{}`))
}
}))
t.Cleanup(srv.Close)
list := func() []listRow {
rows, err := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{
userFilter: "me@example.com", fetchMLflow: true, limit: 10,
}).next(10)
require.NoError(t, err)
require.Len(t, rows, 1)
return rows
}

// A terminal run's links are resolved once, then served from the cache.
runs = []jobs.BaseRun{scanRun(7, jobs.RunLifeCycleStateTerminated, 0)}
first := list()
assert.Equal(t, srv.URL+"/ml/experiments/exp1/runs/m-70/logs", first[0].MLflowURL)
assert.Equal(t, first, list())
assert.Equal(t, 1, outputHits)
assert.Equal(t, 1, mlflowHits)

// A repaired run has a new attempt, so its cached links are stale.
runs = []jobs.BaseRun{scanRun(7, jobs.RunLifeCycleStateTerminated, 0, 1)}
assert.Equal(t, srv.URL+"/ml/experiments/exp1/runs/m-71/logs", list()[0].MLflowURL)
assert.Equal(t, 2, outputHits)

// An active run is never served from the cache.
runs = []jobs.BaseRun{scanRun(8, jobs.RunLifeCycleStateRunning, 0)}
list()
list()
assert.Equal(t, 4, outputHits)

// The scan's entries never stand in for the index path's full rows.
_, _, _, ok := cachedRow(t.Context(), newListCache(t.Context()), srv.URL, 7)
assert.False(t, ok)
}

func TestIsTerminal(t *testing.T) {
assert.True(t, isTerminal(&jobs.Run{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateTerminated}}))
assert.True(t, isTerminal(&jobs.Run{State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateInternalError}}))
Expand Down
Loading