diff --git a/.nextchanges/air/list-index-fallback.md b/.nextchanges/air/list-index-fallback.md new file mode 100644 index 00000000000..ab5fe380913 --- /dev/null +++ b/.nextchanges/air/list-index-fallback.md @@ -0,0 +1 @@ +* `databricks air list --all-status` now warns when the AI Runtime run index is unavailable and it falls back to scanning Jobs runs. That fallback scan is capped at 250 runs, and a stuck index request fails after 15 seconds instead of retrying for minutes. `--all-status --filter user=` now uses the run index instead of scanning the workspace's full run history. ([#7619](https://github.com/databricks/cli/pull/7619)) diff --git a/acceptance/air/list-index-fallback/out.test.toml b/acceptance/air/list-index-fallback/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/air/list-index-fallback/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/air/list-index-fallback/output.txt b/acceptance/air/list-index-fallback/output.txt new file mode 100644 index 00000000000..8cc9da7c7fd --- /dev/null +++ b/acceptance/air/list-index-fallback/output.txt @@ -0,0 +1,30 @@ + +=== list --all-status falls back to a bounded Jobs scan when the index fails (text) + +>>> [CLI] air list --all-status +Warn: air list: the AI Runtime run index is unavailable (failed to list training workflows: index unavailable); falling back to a slower scan of up to 250 recent job runs + Run ID Experiment Status Started Duration Progress MLflow User Accelerators + 334747067049496 qwen-train ● SUCCESS [TIMESTAMP] 12s - qwen-train-001 [USERNAME] 8x H100 + +=== list --all-status fallback (json): the warning stays on stderr +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "runs": [ + { + "run_id": "334747067049496", + "run_name": "qwen-train", + "user": "[USERNAME]", + "status": "SUCCESS", + "started_at": "[TIMESTAMP]", + "is_sweep": false, + "experiment_name": "qwen-train", + "duration_seconds": 12, + "accelerator_type": "GPU_8xH100", + "num_accelerators": 8, + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/334747067049496?o=[WORKSPACE_ID]" + } + ] + } +} diff --git a/acceptance/air/list-index-fallback/script b/acceptance/air/list-index-fallback/script new file mode 100644 index 00000000000..7fbcf8089fc --- /dev/null +++ b/acceptance/air/list-index-fallback/script @@ -0,0 +1,5 @@ +title "list --all-status falls back to a bounded Jobs scan when the index fails (text)" +trace $CLI air list --all-status + +title "list --all-status fallback (json): the warning stays on stderr" +trace $CLI air list --all-status -o json 2>/dev/null diff --git a/acceptance/air/list-index-fallback/test.toml b/acceptance/air/list-index-fallback/test.toml new file mode 100644 index 00000000000..43ee07ce549 --- /dev/null +++ b/acceptance/air/list-index-fallback/test.toml @@ -0,0 +1,49 @@ +# Disable the on-disk run cache so --all-status output is deterministic across runs. +Env.DATABRICKS_CACHE_ENABLED = "false" + +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +# The AiTrainingService index is unavailable, so --all-status falls back to +# scanning Jobs runs/list. +[[Server]] +Pattern = "GET /api/2.0/ai-training/workflows" +Response.StatusCode = 500 +Response.Body = '{"error_code": "INTERNAL_ERROR", "message": "index unavailable"}' + +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/list" +Response.Body = ''' +{ + "runs": [ + { + "run_id": 334747067049496, + "run_name": "qwen-train", + "creator_user_name": "tester@databricks.com", + "start_time": 1717608759000, + "end_time": 1717608771000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "tasks": [{ + "run_id": 334747067049497, + "ai_runtime_task": { + "experiment": "/Users/tester@databricks.com/qwen-train", + "deployments": [{"compute": {"accelerator_type": "GPU_8xH100", "accelerator_count": 8}}] + } + }] + } + ] +} +''' + +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get-output" +Response.Body = ''' +{"ai_runtime_task_output": {"mlflow_experiment_id": "exp1", "mlflow_run_id": "run1"}} +''' + +[[Server]] +Pattern = "GET /api/2.0/mlflow/runs/get" +Response.Body = ''' +{"run": {"info": {"run_name": "qwen-train-001"}}} +''' diff --git a/cmd/air/aitraining.go b/cmd/air/aitraining.go index 704daea6e17..843e97c136e 100644 --- a/cmd/air/aitraining.go +++ b/cmd/air/aitraining.go @@ -10,6 +10,8 @@ import ( "github.com/databricks/cli/libs/auth" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/httpclient" ) // aiTrainingWorkflowsPath is the AiTrainingService index of the caller's own AIR @@ -71,11 +73,31 @@ type aiTrainingWorkflowsResponse struct { NextPageToken string `json:"next_page_token"` } +// The index is an optimization with a Jobs-scan fallback, so a slow or hung +// index page should fail fast rather than ride the SDK defaults (a 60s request +// timeout retried for up to 5 minutes). These bound each page request. +const ( + aiTrainingIndexRequestTimeout = 15 * time.Second + aiTrainingIndexRetryTimeout = 15 * time.Second +) + +// newAiTrainingIndexClient returns an API client for the index with the shorter +// per-request timeout and retry window. It leaves w.Config untouched. +func newAiTrainingIndexClient(w *databricks.WorkspaceClient) (*client.DatabricksClient, error) { + clientCfg, err := config.HTTPClientConfigFromConfig(w.Config) + if err != nil { + return nil, err + } + clientCfg.HTTPTimeout = aiTrainingIndexRequestTimeout + clientCfg.RetryTimeout = aiTrainingIndexRetryTimeout + return client.NewWithClient(w.Config, httpclient.NewApiClient(clientCfg)) +} + // listAiTrainingWorkflows pages the index and returns every workflow ref the // caller owns. Pagination stops at the end or when a page token repeats, which // guards against a stuck or cycling cursor without an arbitrary page cap. func listAiTrainingWorkflows(ctx context.Context, w *databricks.WorkspaceClient, activeOnly bool) ([]workflowRef, error) { - apiClient, err := client.New(w.Config) + apiClient, err := newAiTrainingIndexClient(w) if err != nil { return nil, fmt.Errorf("failed to create API client: %w", err) } diff --git a/cmd/air/list.go b/cmd/air/list.go index 1627470dc8a..a87f05e68b2 100644 --- a/cmd/air/list.go +++ b/cmd/air/list.go @@ -3,6 +3,7 @@ package aircmd import ( "context" "fmt" + "strings" "sync" "github.com/databricks/cli/cmd/root" @@ -21,6 +22,11 @@ import ( // work on a workspace with a large run history. const maxListScan = 2000 +// maxFallbackListScan bounds the Jobs scan that stands in for an unavailable +// AiTrainingService index. That scan can't use active_only, so it walks the full +// run history, which costs several seconds per page on a busy workspace. +const maxFallbackListScan = 250 + // jobsPageLimit is the per-request page size for runs/list; enrichConcurrency // bounds the parallel MLflow lookups. const ( @@ -80,6 +86,10 @@ type listQuery struct { // currentUser resolves the caller's user name for a self-scoped list. It is // nil when an explicit user= filter or --all-users is set. currentUser func() (string, error) + // selfUser is the caller's user name when it was resolved up front, despite + // an explicit user= filter (--all-status only). The strategy compares it + // with userFilter to tell a self filter from a filter for another user. + selfUser string filters listFilters fetchMLflow bool limit int @@ -145,10 +155,26 @@ Filters: // An explicit user= filter wins; otherwise default to the current user // unless --all-users is set. The user is resolved only if a strategy // needs it: the index is already scoped to the caller, and runs/list has - // no creator param, so the scan matches the creator itself. + // no creator param, so the scan matches the creator itself. With + // --all-status the current user is also resolved for an explicit user= + // filter, so filtering on your own email uses the index instead of a + // full-history scan. var currentUser func() (string, error) - if f.User == "" && !allUsers { - currentUser = sync.OnceValues(func() (string, error) { return currentUserEmail(ctx, w) }) + var selfUser string + if !allUsers && (f.User == "" || allStatus) { + if f.User == "" { + currentUser = sync.OnceValues(func() (string, error) { return currentUserEmail(ctx, w) }) + } else { + // Resolve the caller up front so the strategy can tell a self + // filter from a filter for another user. A lookup failure keeps + // the scan, which matches the creator itself. + user, err := currentUserEmail(ctx, w) + if err != nil { + log.Debugf(ctx, "air list: could not resolve current user, scanning Jobs runs for user=%s: %v", f.User, err) + } else { + selfUser = user + } + } } fetcher := newRunFetcher(ctx, w, listQuery{ @@ -156,6 +182,7 @@ Filters: allUsers: allUsers, userFilter: f.User, currentUser: currentUser, + selfUser: selfUser, filters: f, fetchMLflow: root.OutputType(cmd) == flags.OutputText, limit: limit, @@ -234,17 +261,22 @@ func newRunFetcher(ctx context.Context, w *databricks.WorkspaceClient, q listQue // newListStrategy picks the fetch source. The AiTrainingService index serves only // the caller's own runs, so it's used for an all-status self-scoped list; if the // index load fails (e.g. endpoint unavailable in this workspace), we fall back to -// the Jobs scan so the command still returns. Everything else — the default +// a bounded Jobs scan so the command still returns. Everything else — the default // active list, --all-users, and --all-status for another user — uses the scan. func newListStrategy(ctx context.Context, w *databricks.WorkspaceClient, q listQuery) listStrategy { - useIndex := !q.activeOnly && q.currentUser != nil + // The index serves only the caller's own runs: a list with no user filter + // (the scan would resolve the caller itself), or an explicit user= filter + // that matches the caller case-insensitively (resolved up front under + // --all-status). + isSelf := q.userFilter == "" || (q.selfUser != "" && strings.EqualFold(q.userFilter, q.selfUser)) + useIndex := !q.activeOnly && !q.allUsers && isSelf if !useIndex { - return newJobsScanStrategy(ctx, w, q) + return newJobsScanStrategy(ctx, w, q, maxListScan) } idx := newIndexStrategy(ctx, w, q, q.limit) if err := idx.load(); err != nil { - log.Debugf(ctx, "air list: AiTrainingService index unavailable, falling back to Jobs scan: %v", err) - return newJobsScanStrategy(ctx, w, q) + log.Warnf(ctx, "air list: the AI Runtime run index is unavailable (%v); falling back to a slower scan of up to %d recent job runs", err, maxFallbackListScan) + return newJobsScanStrategy(ctx, w, q, maxFallbackListScan) } return idx } @@ -297,10 +329,12 @@ type jobsScanStrategy struct { host string workspaceID int64 + // maxScan caps how many runs this scan inspects; see maxListScan. + maxScan int scanned int } -func newJobsScanStrategy(ctx context.Context, w *databricks.WorkspaceClient, q listQuery) *jobsScanStrategy { +func newJobsScanStrategy(ctx context.Context, w *databricks.WorkspaceClient, q listQuery, maxScan int) *jobsScanStrategy { req := jobs.ListRunsRequest{ RunType: jobs.RunTypeSubmitRun, ExpandTasks: true, @@ -321,12 +355,13 @@ func newJobsScanStrategy(ctx context.Context, w *databricks.WorkspaceClient, q l filters: q.filters, host: w.Config.Host, workspaceID: q.workspaceID, + maxScan: maxScan, } } func (s *jobsScanStrategy) next(want int) ([]listedRun, error) { var entries []listedRun - for len(entries) < want && s.scanned < maxListScan && s.iter.HasNext(s.ctx) { + for len(entries) < want && s.scanned < s.maxScan && s.iter.HasNext(s.ctx) { base, err := s.iter.Next(s.ctx) if err != nil { return nil, fmt.Errorf("failed to list runs: %w", err) @@ -356,19 +391,26 @@ func (s *jobsScanStrategy) next(want int) ([]listedRun, error) { } func (s *jobsScanStrategy) done() bool { - return s.scanned >= maxListScan || !s.iter.HasNext(s.ctx) + return s.scanned >= s.maxScan || !s.iter.HasNext(s.ctx) } func (s *jobsScanStrategy) truncated() bool { - return s.scanned >= maxListScan + return s.scanned >= s.maxScan } -// warnIfTruncated logs when a scan hit its safety cap, so one-shot output signals +// warnIfTruncated warns when a scan hit its safety cap, so the output signals // its results may be incomplete. func warnIfTruncated(ctx context.Context, f *runFetcher) { - if f.strategy.truncated() { - log.Warnf(ctx, "air list: stopped after scanning %d runs; results may be incomplete", maxListScan) + if !f.strategy.truncated() { + return + } + // Only the scan strategy can report truncated, and it carries the scanned + // count for the warning. + s, ok := f.strategy.(*jobsScanStrategy) + if !ok { + return } + log.Warnf(ctx, "air list: stopped after scanning %d runs; results may be incomplete. Narrow the search with --filter or a smaller --limit", s.scanned) } // setMLflowLinks fills in each row's MLflow link, label, and experiment URL - diff --git a/cmd/air/list_index_test.go b/cmd/air/list_index_test.go index e6ed2d0df41..0d4f37887c0 100644 --- a/cmd/air/list_index_test.go +++ b/cmd/air/list_index_test.go @@ -1,14 +1,21 @@ package aircmd import ( + "bytes" + "context" "encoding/json" "fmt" + "log/slog" "net/http" "net/http/httptest" "strconv" "sync/atomic" "testing" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -172,6 +179,8 @@ func TestNewListStrategyGate(t *testing.T) { {"all-status self → index", listQuery{currentUser: selfUser, limit: 5}, true}, {"all-status all-users → scan", listQuery{allUsers: true}, false}, {"all-status other user → scan", listQuery{userFilter: "other@example.com"}, false}, + {"all-status self filter, other case → index", listQuery{userFilter: "Me@Example.com", selfUser: "me@example.com", limit: 5}, true}, + {"all-status filter, current user unknown → scan", listQuery{userFilter: "me@example.com"}, false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -192,7 +201,7 @@ func TestNewListStrategyGate(t *testing.T) { } func TestNewListStrategyFallsBackWhenIndexFails(t *testing.T) { - // Index 500 must silently fall back to the Jobs scan, not fail the command. + // Index 500 must fall back to the Jobs scan, not fail the command. srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == aiTrainingWorkflowsPath { w.WriteHeader(http.StatusInternalServerError) @@ -236,3 +245,121 @@ func TestIndexStrategySkipsUserLookup(t *testing.T) { require.Len(t, rows, 1) assert.Zero(t, lookups.Load()) } + +// captureWarnings routes warnings logged through ctx into a buffer. +func captureWarnings(ctx context.Context) (context.Context, *bytes.Buffer) { + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelWarn})) + return log.NewContext(ctx, logger), &buf +} + +// endlessNonAirRunsPage is a full runs/list page of non-AIR runs that always +// points at another page, standing in for a deep run history. +func endlessNonAirRunsPage(t *testing.T) string { + runs := make([]jobs.BaseRun, jobsPageLimit) + for i := range runs { + runs[i] = jobs.BaseRun{RunId: int64(i + 1), CreatorUserName: "me@example.com", Tasks: []jobs.RunTask{{}}} + } + return runsListBody(t, "more", runs...) +} + +func TestFallbackScanWarnsAndIsBounded(t *testing.T) { + var listCalls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case aiTrainingWorkflowsPath: + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"message":"boom"}`)) + case "/api/2.2/jobs/runs/list": + listCalls.Add(1) + assert.Empty(t, r.URL.Query().Get("active_only")) + _, _ = fmt.Fprint(w, endlessNonAirRunsPage(t)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + t.Setenv("DATABRICKS_CACHE_ENABLED", "false") + + var stdout bytes.Buffer + ctx, warnings := captureWarnings(t.Context()) + ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputJSON, nil, &stdout, &stdout, "", "")) + f := newRunFetcher(ctx, newTestWorkspaceClient(t, srv.URL), listQuery{ + userFilter: "me@example.com", selfUser: "me@example.com", limit: 10, + }) + rows, err := f.next(10) + require.NoError(t, err) + assert.Empty(t, rows) + warnIfTruncated(ctx, f) + + // The fallback stops at its own cap instead of maxListScan. + assert.Equal(t, int32(maxFallbackListScan/jobsPageLimit), listCalls.Load()) + assert.Contains(t, warnings.String(), "run index is unavailable") + assert.Contains(t, warnings.String(), "falling back to a slower scan of up to 250 recent job runs") + assert.Contains(t, warnings.String(), "stopped after scanning 250 runs; results may be incomplete") + assert.Empty(t, stdout.String()) +} + +func TestRegularScanKeepsFullCap(t *testing.T) { + var listCalls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/2.2/jobs/runs/list" { + listCalls.Add(1) + _, _ = fmt.Fprint(w, endlessNonAirRunsPage(t)) + return + } + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + + ctx, warnings := captureWarnings(t.Context()) + f := newRunFetcher(ctx, newTestWorkspaceClient(t, srv.URL), listQuery{allUsers: true, limit: 10}) + _, err := f.next(10) + require.NoError(t, err) + warnIfTruncated(ctx, f) + + assert.Equal(t, int32(maxListScan/jobsPageLimit), listCalls.Load()) + assert.NotContains(t, warnings.String(), "run index is unavailable") + assert.Contains(t, warnings.String(), "stopped after scanning 2000 runs; results may be incomplete") +} + +// TestListSelfUserFilterUsesIndex pins that --all-status --filter user= is served by the index, like the unfiltered self list, rather than a +// full-history Jobs scan. +func TestListSelfUserFilterUsesIndex(t *testing.T) { + var indexHits, listHits atomic.Int32 + run := indexRun(7, 3000_000) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/2.0/preview/scim/v2/Me": + _, _ = w.Write([]byte(`{"userName":"me@example.com"}`)) + case aiTrainingWorkflowsPath: + indexHits.Add(1) + _, _ = w.Write([]byte(`{"training_workflows":[{"job_run_id":"7","submit_time":{"seconds":3000}}]}`)) + case "/api/2.2/jobs/runs/get": + _ = json.NewEncoder(w).Encode(run) + case "/api/2.2/jobs/runs/list": + listHits.Add(1) + _, _ = w.Write([]byte(`{}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + t.Setenv("DATABRICKS_CACHE_ENABLED", "false") + + var stdout bytes.Buffer + ctx, warnings := captureWarnings(t.Context()) + ctx = cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srv.URL)) + ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputJSON, nil, &stdout, &stdout, "", "")) + cmd := withOutput(newListCommand(), flags.OutputJSON) + cmd.SetContext(ctx) + require.NoError(t, cmd.Flags().Set("all-status", "true")) + require.NoError(t, cmd.Flags().Set("filter", "user=me@example.com")) + + require.NoError(t, cmd.RunE(cmd, nil)) + assert.Equal(t, int32(1), indexHits.Load()) + assert.Equal(t, int32(0), listHits.Load()) + assert.Contains(t, stdout.String(), `"run_id": "7"`) + assert.Empty(t, warnings.String()) +} diff --git a/cmd/air/list_tui.go b/cmd/air/list_tui.go index af5951bca44..3245eee8f90 100644 --- a/cmd/air/list_tui.go +++ b/cmd/air/list_tui.go @@ -46,7 +46,11 @@ func renderListText(cmd *cobra.Command, f *runFetcher, limit int, activeOnly boo tea.WithInput(cmd.InOrStdin()), tea.WithOutput(out), ).Run() - return err + if err != nil { + return err + } + warnIfTruncated(ctx, f) + return nil } rows, err := f.next(limit)