Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/7615.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `air list` no longer waits on extra current-user requests before listing runs, which makes the default list about 150ms faster. ([#7615](https://github.com/databricks/cli/pull/7615))
58 changes: 30 additions & 28 deletions cmd/air/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@ package aircmd
import (
"context"
"fmt"
"sync"

"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/cmdctx"
"github.com/databricks/cli/libs/flags"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/listing"
"github.com/databricks/databricks-sdk-go/service/iam"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/spf13/cobra"
"golang.org/x/sync/errgroup"
Expand Down Expand Up @@ -74,10 +74,12 @@ type listedRun struct {

// listQuery holds the resolved inputs to a runFetcher.
type listQuery struct {
activeOnly bool
allUsers bool
userFilter string
currentUser string
activeOnly bool
allUsers bool
userFilter string
// currentUser resolves the caller's user name for a self-scoped list. It is
// nil when an explicit user= filter or --all-users is set.
currentUser func() (string, error)
filters listFilters
fetchMLflow bool
limit int
Expand Down Expand Up @@ -141,37 +143,23 @@ Filters:
}

// An explicit user= filter wins; otherwise default to the current user
// unless --all-users is set. runs/list has no creator param, so the
// creator is matched while scanning.
userFilter := f.User
var currentUser string
if userFilter == "" && !allUsers {
me, err := w.CurrentUser.Me(ctx, iam.MeRequest{})
if err != nil {
return fmt.Errorf("failed to resolve current user: %w", err)
}
currentUser = me.UserName
userFilter = currentUser
}

// Fetch workspace ID once for dashboard links; proceed with 0 on error.
var workspaceID int64
wsID, err := w.CurrentWorkspaceID(ctx)
if err != nil {
log.Debugf(ctx, "air list: could not fetch workspace ID for dashboard links: %v", err)
} else {
workspaceID = wsID
// unless --all-users is set. The user is resolved only if a strategy
// needs it: the index is already scoped to the caller, and runs/list has
// no creator param, so the scan matches the creator itself.
var currentUser func() (string, error)
if f.User == "" && !allUsers {
currentUser = sync.OnceValues(func() (string, error) { return currentUserEmail(ctx, w) })
}

fetcher := newRunFetcher(ctx, w, listQuery{
activeOnly: !allStatus,
allUsers: allUsers,
userFilter: userFilter,
userFilter: f.User,
currentUser: currentUser,
filters: f,
fetchMLflow: root.OutputType(cmd) == flags.OutputText,
limit: limit,
workspaceID: workspaceID,
workspaceID: workspaceIDForRunLink(ctx, w),
})
if f.AcceleratorType != "" {
fetcher.validateEmpty = func() error {
Expand Down Expand Up @@ -249,7 +237,7 @@ func newRunFetcher(ctx context.Context, w *databricks.WorkspaceClient, q listQue
// the Jobs scan so the command still returns. Everything else — the default
// active list, --all-users, and --all-status for another user — uses the scan.
func newListStrategy(ctx context.Context, w *databricks.WorkspaceClient, q listQuery) listStrategy {
useIndex := !q.activeOnly && !q.allUsers && (q.userFilter == "" || q.userFilter == q.currentUser)
useIndex := !q.activeOnly && q.currentUser != nil
if !useIndex {
return newJobsScanStrategy(ctx, w, q)
}
Expand Down Expand Up @@ -304,6 +292,7 @@ type jobsScanStrategy struct {
w *databricks.WorkspaceClient
iter listing.Iterator[jobs.BaseRun]
userFilter string
currentUser func() (string, error) // resolves userFilter for a self-scoped scan
filters listFilters
host string
workspaceID int64
Expand All @@ -318,11 +307,17 @@ func newJobsScanStrategy(ctx context.Context, w *databricks.WorkspaceClient, q l
Limit: jobsPageLimit,
ActiveOnly: q.activeOnly,
}
// Resolve the current user while the first runs/list page is fetched; next()
// waits for it only when it first needs to match a run's creator.
if q.currentUser != nil {
go func() { _, _ = q.currentUser() }()
}
return &jobsScanStrategy{
ctx: ctx,
w: w,
iter: w.Jobs.ListRuns(ctx, req),
userFilter: q.userFilter,
currentUser: q.currentUser,
filters: q.filters,
host: w.Config.Host,
workspaceID: q.workspaceID,
Expand All @@ -342,6 +337,13 @@ func (s *jobsScanStrategy) next(want int) ([]listedRun, error) {
if !isAirRun(run) {
continue
}
if s.currentUser != nil {
user, err := s.currentUser()
if err != nil {
return nil, err
}
s.userFilter, s.currentUser = user, nil
}
if s.userFilter != "" && run.CreatorUserName != s.userFilter {
continue
}
Expand Down
8 changes: 4 additions & 4 deletions cmd/air/list_cache_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func TestIndexStrategyServesCachedRowWithoutFetch(t *testing.T) {
putRow(ctx, newListCache(ctx), host, 7, []jobs.RunTask{{RunId: 70}}, 1000_000, listRow{RunID: "7", Status: "SUCCESS"}, filterFields{})

f := newRunFetcher(ctx, newTestWorkspaceClient(t, host), listQuery{
userFilter: "me@example.com", currentUser: "me@example.com", limit: 10,
currentUser: selfUser, limit: 10,
})
rows, err := f.next(10)
require.NoError(t, err)
Expand All @@ -82,7 +82,7 @@ func TestIndexStrategyFiltersCachedRow(t *testing.T) {
filterFields{Experiment: "bar"})

f := newRunFetcher(ctx, newTestWorkspaceClient(t, host), listQuery{
userFilter: "me@example.com", currentUser: "me@example.com", limit: 10,
currentUser: selfUser, limit: 10,
filters: listFilters{Experiment: "foo"},
})
rows, err := f.next(10)
Expand All @@ -104,7 +104,7 @@ func TestIndexStrategyServesMatchingCachedRow(t *testing.T) {
filterFields{Experiment: "foo"})

f := newRunFetcher(ctx, newTestWorkspaceClient(t, host), listQuery{
userFilter: "me@example.com", currentUser: "me@example.com", limit: 10,
currentUser: selfUser, limit: 10,
filters: listFilters{Experiment: "foo"},
})
rows, err := f.next(10)
Expand Down Expand Up @@ -149,7 +149,7 @@ func TestIndexStrategyCachesMLflowEnrichment(t *testing.T) {
t.Cleanup(srv.Close)

query := listQuery{
userFilter: "me@example.com", currentUser: "me@example.com", limit: 10,
currentUser: selfUser, limit: 10,
}
rows, err := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), query).next(10)
require.NoError(t, err)
Expand Down
5 changes: 1 addition & 4 deletions cmd/air/list_detail.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,8 @@ func runDetailText(ctx context.Context, w *databricks.WorkspaceClient, runID int
return "", fmt.Errorf("failed to fetch run: %w", err)
}

// A missing workspace id only drops the ?o= org hint from the dashboard link.
workspaceID, _ := w.CurrentWorkspaceID(ctx)

data := buildGetData(run)
data.DashboardURL = dashboardURL(w.Config.Host, runID, workspaceID)
data.DashboardURL = dashboardURL(w.Config.Host, runID, workspaceIDForRunLink(ctx, w))
ids := mlflowIDs(ctx, w, run)
if ids != nil {
url := mlflowLogsURL(w.Config.Host, ids)
Expand Down
4 changes: 2 additions & 2 deletions cmd/air/list_detail_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ func TestListRowAndPanesLinkTheSameOlderRetry(t *testing.T) {

// When the list, the info pane and the logs pane render the run.
rows, err := newRunFetcher(ctx, w, listQuery{
userFilter: "me@example.com", currentUser: "me@example.com", limit: 10, fetchMLflow: true,
currentUser: selfUser, limit: 10, fetchMLflow: true,
}).next(10)
require.NoError(t, err)
info, err := runDetailText(ctx, w, 7)
Expand Down Expand Up @@ -132,7 +132,7 @@ func TestUnnamedOlderRetryLabelFitsTheListColumnAndIsWholeInTheInfoPane(t *testi

// When the list and the info pane render the run.
rows, err := newRunFetcher(ctx, w, listQuery{
userFilter: "me@example.com", currentUser: "me@example.com", limit: 10, fetchMLflow: true,
currentUser: selfUser, limit: 10, fetchMLflow: true,
}).next(10)
require.NoError(t, err)
info, err := runDetailText(ctx, w, 7)
Expand Down
45 changes: 35 additions & 10 deletions cmd/air/list_index_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"net/http"
"net/http/httptest"
"strconv"
"sync/atomic"
"testing"

"github.com/databricks/databricks-sdk-go/service/jobs"
Expand Down Expand Up @@ -62,6 +63,9 @@ func indexAndGetServer(t *testing.T, refs []workflowRef, runs map[int64]jobs.Run
return srv, hits
}

// selfUser resolves the current user for a self-scoped listQuery.
func selfUser() (string, error) { return "me@example.com", nil }

func TestIndexStrategyOrdersAndLimits(t *testing.T) {
// Three runs, out of submit-time order; newest two should win, newest-first.
refs := []workflowRef{
Expand All @@ -78,7 +82,7 @@ func TestIndexStrategyOrdersAndLimits(t *testing.T) {
t.Setenv("DATABRICKS_CACHE_ENABLED", "false")

f := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{
userFilter: "me@example.com", currentUser: "me@example.com", limit: 2,
currentUser: selfUser, limit: 2,
})
rows, err := f.next(10)
require.NoError(t, err)
Expand All @@ -103,7 +107,7 @@ func TestIndexStrategyOverFetchesWithTaskFilter(t *testing.T) {
t.Setenv("DATABRICKS_CACHE_ENABLED", "false")

f := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{
userFilter: "me@example.com", currentUser: "me@example.com", limit: 1,
currentUser: selfUser, limit: 1,
filters: listFilters{Experiment: "qwen"},
})
rows, err := f.next(1)
Expand All @@ -123,7 +127,7 @@ func TestIndexStrategyDropsForbiddenAndMissing(t *testing.T) {
t.Setenv("DATABRICKS_CACHE_ENABLED", "false")

f := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{
userFilter: "me@example.com", currentUser: "me@example.com", limit: 10,
currentUser: selfUser, limit: 10,
})
rows, err := f.next(10)
require.NoError(t, err)
Expand Down Expand Up @@ -151,7 +155,7 @@ func TestIndexStrategyPropagatesServerError(t *testing.T) {
t.Setenv("DATABRICKS_CACHE_ENABLED", "false")

f := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{
userFilter: "me@example.com", currentUser: "me@example.com", limit: 10,
currentUser: selfUser, limit: 10,
})
_, err := f.next(10)
require.Error(t, err) // 500 is systemic, not an ACL drop
Expand All @@ -164,10 +168,10 @@ func TestNewListStrategyGate(t *testing.T) {
q listQuery
wantIndex bool
}{
{"active default → scan", listQuery{activeOnly: true, userFilter: "me@example.com", currentUser: "me@example.com"}, false},
{"all-status self → index", listQuery{userFilter: "me@example.com", currentUser: "me@example.com", limit: 5}, true},
{"active default → scan", listQuery{activeOnly: true, currentUser: selfUser}, false},
{"all-status self → index", listQuery{currentUser: selfUser, limit: 5}, true},
{"all-status all-users → scan", listQuery{allUsers: true}, false},
{"all-status other user → scan", listQuery{userFilter: "other@example.com", currentUser: "me@example.com"}, false},
{"all-status other user → scan", listQuery{userFilter: "other@example.com"}, false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
Expand Down Expand Up @@ -196,7 +200,9 @@ func TestNewListStrategyFallsBackWhenIndexFails(t *testing.T) {
return
}
if r.URL.Path == "/api/2.2/jobs/runs/list" {
_, _ = fmt.Fprint(w, runsListBody(t, "", airBaseRun(1, "me@example.com", "GPU_1xH100", 1, "exp")))
_, _ = fmt.Fprint(w, runsListBody(t, "",
airBaseRun(1, "me@example.com", "GPU_1xH100", 1, "exp"),
airBaseRun(2, "other@example.com", "GPU_1xH100", 1, "exp")))
return
}
_, _ = w.Write([]byte(`{}`))
Expand All @@ -205,9 +211,28 @@ func TestNewListStrategyFallsBackWhenIndexFails(t *testing.T) {
t.Setenv("DATABRICKS_CACHE_ENABLED", "false")

f := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{
userFilter: "me@example.com", currentUser: "me@example.com", limit: 10,
currentUser: selfUser, limit: 10,
})
rows, err := f.next(10)
require.NoError(t, err)
require.Len(t, rows, 1) // served by the Jobs scan fallback
// Served by the Jobs scan fallback, still scoped to the current user.
require.Len(t, rows, 1)
assert.Equal(t, "1", rows[0].RunID)
}

func TestIndexStrategySkipsUserLookup(t *testing.T) {
refs := []workflowRef{{jobRunID: 1, submitTimeMs: 1000_000}}
srv, _ := indexAndGetServer(t, refs, map[int64]jobs.Run{1: indexRun(1, 1000_000)}, nil, nil)
t.Setenv("DATABRICKS_CACHE_ENABLED", "false")

// The index is already scoped to the caller, so it never needs the user name.
var lookups atomic.Int32
f := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{
currentUser: func() (string, error) { lookups.Add(1); return "me@example.com", nil },
limit: 10,
})
rows, err := f.next(10)
require.NoError(t, err)
require.Len(t, rows, 1)
assert.Zero(t, lookups.Load())
}
38 changes: 38 additions & 0 deletions cmd/air/list_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,12 @@ package aircmd

import (
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strconv"
"sync"
"sync/atomic"
"testing"

"github.com/databricks/cli/libs/cmdctx"
Expand Down Expand Up @@ -84,6 +87,41 @@ func TestListAirRunsFiltersUserAndType(t *testing.T) {
assert.Equal(t, "5", rows[1].RunID)
}

func TestListAirRunsResolvesCurrentUserOnce(t *testing.T) {
runs := []jobs.BaseRun{
airBaseRun(1, "me@example.com", "GPU_1xA10", 1, "exp"),
airBaseRun(2, "other@example.com", "GPU_1xA10", 1, "exp"),
airBaseRun(3, "me@example.com", "GPU_1xA10", 1, "exp"),
}
srv := runsServer(t, runsListBody(t, "", runs...))

var lookups atomic.Int32
rows, err := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{
activeOnly: true,
currentUser: sync.OnceValues(func() (string, error) {
lookups.Add(1)
return "me@example.com", nil
}),
}).next(10)
require.NoError(t, err)
require.Len(t, rows, 2)
assert.Equal(t, "1", rows[0].RunID)
assert.Equal(t, "3", rows[1].RunID)
assert.Equal(t, int32(1), lookups.Load())
}

func TestListAirRunsReportsUserLookupFailure(t *testing.T) {
srv := runsServer(t, runsListBody(t, "", airBaseRun(1, "me@example.com", "GPU_1xA10", 1, "exp")))

_, err := newRunFetcher(t.Context(), newTestWorkspaceClient(t, srv.URL), listQuery{
activeOnly: true,
currentUser: sync.OnceValues(func() (string, error) {
return "", errors.New("failed to resolve current user: boom")
}),
}).next(10)
require.ErrorContains(t, err, "failed to resolve current user: boom")
}

func TestListAirRunsExperimentFilter(t *testing.T) {
runs := []jobs.BaseRun{
airBaseRun(1, "me@example.com", "GPU_1xH100", 1, "/Users/me@example.com/qwen-train"),
Expand Down
4 changes: 2 additions & 2 deletions cmd/air/runsubmit.go
Original file line number Diff line number Diff line change
Expand Up @@ -468,12 +468,12 @@ func workspaceIDForRunLink(ctx context.Context, w *databricks.WorkspaceClient) i
defer cancel()
id, err := auth.ResolveWorkspaceID(ctx, w)
if err != nil {
log.Debugf(ctx, "air run: could not resolve workspace ID for the run link: %v", err)
log.Debugf(ctx, "could not resolve workspace ID for the run link: %v", err)
return 0
}
workspaceID, err := strconv.ParseInt(id, 10, 64)
if err != nil {
log.Debugf(ctx, "air run: workspace ID %q for the run link is not numeric", id)
log.Debugf(ctx, "workspace ID %q for the run link is not numeric", id)
return 0
}
return workspaceID
Expand Down
Loading