From 6f9063b5cb932498e59aecfd5fe8cf7b13322a0d Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Fri, 13 Nov 2026 01:28:38 +0000 Subject: [PATCH 1/2] air: Use the run index for a self user filter and bound the index fallback in air list `air list --all-status --filter user=` skipped the AiTrainingService index because the current user was only resolved when no user filter was set, so it scanned the workspace's full Jobs run history (minutes on a busy workspace). Resolve the current user for --all-status and treat a case-insensitive match as self, so that filter uses the index. When the index load fails, warn instead of logging at debug, and cap the Jobs scan that replaces it at 250 runs (it can't use active_only, so it walks the full history). Give index requests a 15 s request timeout and retry window so a hung index fails fast instead of waiting 60 s per page. Warn after the interactive table exits when a scan hit its cap. Co-authored-by: Isaac --- .nextchanges/air/list-index-fallback.md | 1 + .../air/list-index-fallback/out.test.toml | 1 + acceptance/air/list-index-fallback/output.txt | 25 ++++ acceptance/air/list-index-fallback/script | 5 + acceptance/air/list-index-fallback/test.toml | 49 +++++++ cmd/air/aitraining.go | 24 +++- cmd/air/list.go | 67 +++++++-- cmd/air/list_index_test.go | 129 +++++++++++++++++- cmd/air/list_tui.go | 6 +- 9 files changed, 289 insertions(+), 18 deletions(-) create mode 100644 .nextchanges/air/list-index-fallback.md create mode 100644 acceptance/air/list-index-fallback/out.test.toml create mode 100644 acceptance/air/list-index-fallback/output.txt create mode 100644 acceptance/air/list-index-fallback/script create mode 100644 acceptance/air/list-index-fallback/test.toml diff --git a/.nextchanges/air/list-index-fallback.md b/.nextchanges/air/list-index-fallback.md new file mode 100644 index 00000000000..127dd497b7d --- /dev/null +++ b/.nextchanges/air/list-index-fallback.md @@ -0,0 +1 @@ +* `databricks air list --all-status` now warns when the AI Runtime run index is unavailable and it falls back to scanning Jobs runs. That fallback scan is capped at 250 runs, and a stuck index request fails after 15 seconds instead of retrying for minutes. `--all-status --filter user=` now uses the run index instead of scanning the workspace's full run history. diff --git a/acceptance/air/list-index-fallback/out.test.toml b/acceptance/air/list-index-fallback/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/air/list-index-fallback/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/air/list-index-fallback/output.txt b/acceptance/air/list-index-fallback/output.txt new file mode 100644 index 00000000000..9460c16bc81 --- /dev/null +++ b/acceptance/air/list-index-fallback/output.txt @@ -0,0 +1,25 @@ + +=== list --all-status falls back to a bounded Jobs scan when the index fails (text) + +>>> [CLI] air list --all-status +Warn: air list: the AI Runtime run index is unavailable (failed to list training workflows: index unavailable); falling back to a slower scan of up to 250 recent job runs + Run ID Experiment Status Started Duration Progress MLflow User Accelerators + 334747067049496 qwen-train ● SUCCESS [TIMESTAMP] 12s - qwen-train-001 [USERNAME] 8x H100 + +=== list --all-status fallback (json): the warning stays on stderr +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "runs": [ + { + "run_id": "334747067049496", + "run_name": "qwen-train", + "user": "[USERNAME]", + "status": "SUCCESS", + "started_at": "[TIMESTAMP]", + "is_sweep": false + } + ] + } +} diff --git a/acceptance/air/list-index-fallback/script b/acceptance/air/list-index-fallback/script new file mode 100644 index 00000000000..7fbcf8089fc --- /dev/null +++ b/acceptance/air/list-index-fallback/script @@ -0,0 +1,5 @@ +title "list --all-status falls back to a bounded Jobs scan when the index fails (text)" +trace $CLI air list --all-status + +title "list --all-status fallback (json): the warning stays on stderr" +trace $CLI air list --all-status -o json 2>/dev/null diff --git a/acceptance/air/list-index-fallback/test.toml b/acceptance/air/list-index-fallback/test.toml new file mode 100644 index 00000000000..43ee07ce549 --- /dev/null +++ b/acceptance/air/list-index-fallback/test.toml @@ -0,0 +1,49 @@ +# Disable the on-disk run cache so --all-status output is deterministic across runs. +Env.DATABRICKS_CACHE_ENABLED = "false" + +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +# The AiTrainingService index is unavailable, so --all-status falls back to +# scanning Jobs runs/list. +[[Server]] +Pattern = "GET /api/2.0/ai-training/workflows" +Response.StatusCode = 500 +Response.Body = '{"error_code": "INTERNAL_ERROR", "message": "index unavailable"}' + +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/list" +Response.Body = ''' +{ + "runs": [ + { + "run_id": 334747067049496, + "run_name": "qwen-train", + "creator_user_name": "tester@databricks.com", + "start_time": 1717608759000, + "end_time": 1717608771000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "tasks": [{ + "run_id": 334747067049497, + "ai_runtime_task": { + "experiment": "/Users/tester@databricks.com/qwen-train", + "deployments": [{"compute": {"accelerator_type": "GPU_8xH100", "accelerator_count": 8}}] + } + }] + } + ] +} +''' + +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get-output" +Response.Body = ''' +{"ai_runtime_task_output": {"mlflow_experiment_id": "exp1", "mlflow_run_id": "run1"}} +''' + +[[Server]] +Pattern = "GET /api/2.0/mlflow/runs/get" +Response.Body = ''' +{"run": {"info": {"run_name": "qwen-train-001"}}} +''' diff --git a/cmd/air/aitraining.go b/cmd/air/aitraining.go index 704daea6e17..843e97c136e 100644 --- a/cmd/air/aitraining.go +++ b/cmd/air/aitraining.go @@ -10,6 +10,8 @@ import ( "github.com/databricks/cli/libs/auth" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/httpclient" ) // aiTrainingWorkflowsPath is the AiTrainingService index of the caller's own AIR @@ -71,11 +73,31 @@ type aiTrainingWorkflowsResponse struct { NextPageToken string `json:"next_page_token"` } +// The index is an optimization with a Jobs-scan fallback, so a slow or hung +// index page should fail fast rather than ride the SDK defaults (a 60s request +// timeout retried for up to 5 minutes). These bound each page request. +const ( + aiTrainingIndexRequestTimeout = 15 * time.Second + aiTrainingIndexRetryTimeout = 15 * time.Second +) + +// newAiTrainingIndexClient returns an API client for the index with the shorter +// per-request timeout and retry window. It leaves w.Config untouched. +func newAiTrainingIndexClient(w *databricks.WorkspaceClient) (*client.DatabricksClient, error) { + clientCfg, err := config.HTTPClientConfigFromConfig(w.Config) + if err != nil { + return nil, err + } + clientCfg.HTTPTimeout = aiTrainingIndexRequestTimeout + clientCfg.RetryTimeout = aiTrainingIndexRetryTimeout + return client.NewWithClient(w.Config, httpclient.NewApiClient(clientCfg)) +} + // listAiTrainingWorkflows pages the index and returns every workflow ref the // caller owns. Pagination stops at the end or when a page token repeats, which // guards against a stuck or cycling cursor without an arbitrary page cap. func listAiTrainingWorkflows(ctx context.Context, w *databricks.WorkspaceClient, activeOnly bool) ([]workflowRef, error) { - apiClient, err := client.New(w.Config) + apiClient, err := newAiTrainingIndexClient(w) if err != nil { return nil, fmt.Errorf("failed to create API client: %w", err) } diff --git a/cmd/air/list.go b/cmd/air/list.go index 1627470dc8a..b9d991c9bea 100644 --- a/cmd/air/list.go +++ b/cmd/air/list.go @@ -3,6 +3,7 @@ package aircmd import ( "context" "fmt" + "strings" "sync" "github.com/databricks/cli/cmd/root" @@ -21,6 +22,11 @@ import ( // work on a workspace with a large run history. const maxListScan = 2000 +// maxFallbackListScan bounds the Jobs scan that stands in for an unavailable +// AiTrainingService index. That scan can't use active_only, so it walks the full +// run history, which costs several seconds per page on a busy workspace. +const maxFallbackListScan = 250 + // jobsPageLimit is the per-request page size for runs/list; enrichConcurrency // bounds the parallel MLflow lookups. const ( @@ -80,6 +86,10 @@ type listQuery struct { // currentUser resolves the caller's user name for a self-scoped list. It is // nil when an explicit user= filter or --all-users is set. currentUser func() (string, error) + // selfUser is the caller's user name when it was resolved up front, despite + // an explicit user= filter (--all-status only). The strategy compares it + // with userFilter to tell a self filter from a filter for another user. + selfUser string filters listFilters fetchMLflow bool limit int @@ -145,10 +155,26 @@ Filters: // An explicit user= filter wins; otherwise default to the current user // unless --all-users is set. The user is resolved only if a strategy // needs it: the index is already scoped to the caller, and runs/list has - // no creator param, so the scan matches the creator itself. + // no creator param, so the scan matches the creator itself. With + // --all-status the current user is also resolved for an explicit user= + // filter, so filtering on your own email uses the index instead of a + // full-history scan. var currentUser func() (string, error) - if f.User == "" && !allUsers { - currentUser = sync.OnceValues(func() (string, error) { return currentUserEmail(ctx, w) }) + var selfUser string + if !allUsers && (f.User == "" || allStatus) { + if f.User == "" { + currentUser = sync.OnceValues(func() (string, error) { return currentUserEmail(ctx, w) }) + } else { + // Resolve the caller up front so the strategy can tell a self + // filter from a filter for another user. A lookup failure keeps + // the scan, which matches the creator itself. + user, err := currentUserEmail(ctx, w) + if err != nil { + log.Debugf(ctx, "air list: could not resolve current user, scanning Jobs runs for user=%s: %v", f.User, err) + } else { + selfUser = user + } + } } fetcher := newRunFetcher(ctx, w, listQuery{ @@ -156,6 +182,7 @@ Filters: allUsers: allUsers, userFilter: f.User, currentUser: currentUser, + selfUser: selfUser, filters: f, fetchMLflow: root.OutputType(cmd) == flags.OutputText, limit: limit, @@ -234,17 +261,22 @@ func newRunFetcher(ctx context.Context, w *databricks.WorkspaceClient, q listQue // newListStrategy picks the fetch source. The AiTrainingService index serves only // the caller's own runs, so it's used for an all-status self-scoped list; if the // index load fails (e.g. endpoint unavailable in this workspace), we fall back to -// the Jobs scan so the command still returns. Everything else — the default +// a bounded Jobs scan so the command still returns. Everything else — the default // active list, --all-users, and --all-status for another user — uses the scan. func newListStrategy(ctx context.Context, w *databricks.WorkspaceClient, q listQuery) listStrategy { - useIndex := !q.activeOnly && q.currentUser != nil + // The index serves only the caller's own runs: a list with no user filter + // (the scan would resolve the caller itself), or an explicit user= filter + // that matches the caller case-insensitively (resolved up front under + // --all-status). + isSelf := q.userFilter == "" || (q.selfUser != "" && strings.EqualFold(q.userFilter, q.selfUser)) + useIndex := !q.activeOnly && !q.allUsers && isSelf if !useIndex { - return newJobsScanStrategy(ctx, w, q) + return newJobsScanStrategy(ctx, w, q, maxListScan) } idx := newIndexStrategy(ctx, w, q, q.limit) if err := idx.load(); err != nil { - log.Debugf(ctx, "air list: AiTrainingService index unavailable, falling back to Jobs scan: %v", err) - return newJobsScanStrategy(ctx, w, q) + log.Warnf(ctx, "air list: the AI Runtime run index is unavailable (%v); falling back to a slower scan of up to %d recent job runs", err, maxFallbackListScan) + return newJobsScanStrategy(ctx, w, q, maxFallbackListScan) } return idx } @@ -297,10 +329,12 @@ type jobsScanStrategy struct { host string workspaceID int64 + // maxScan caps how many runs this scan inspects; see maxListScan. + maxScan int scanned int } -func newJobsScanStrategy(ctx context.Context, w *databricks.WorkspaceClient, q listQuery) *jobsScanStrategy { +func newJobsScanStrategy(ctx context.Context, w *databricks.WorkspaceClient, q listQuery, maxScan int) *jobsScanStrategy { req := jobs.ListRunsRequest{ RunType: jobs.RunTypeSubmitRun, ExpandTasks: true, @@ -321,12 +355,13 @@ func newJobsScanStrategy(ctx context.Context, w *databricks.WorkspaceClient, q l filters: q.filters, host: w.Config.Host, workspaceID: q.workspaceID, + maxScan: maxScan, } } func (s *jobsScanStrategy) next(want int) ([]listedRun, error) { var entries []listedRun - for len(entries) < want && s.scanned < maxListScan && s.iter.HasNext(s.ctx) { + for len(entries) < want && s.scanned < s.maxScan && s.iter.HasNext(s.ctx) { base, err := s.iter.Next(s.ctx) if err != nil { return nil, fmt.Errorf("failed to list runs: %w", err) @@ -356,19 +391,21 @@ func (s *jobsScanStrategy) next(want int) ([]listedRun, error) { } func (s *jobsScanStrategy) done() bool { - return s.scanned >= maxListScan || !s.iter.HasNext(s.ctx) + return s.scanned >= s.maxScan || !s.iter.HasNext(s.ctx) } func (s *jobsScanStrategy) truncated() bool { - return s.scanned >= maxListScan + return s.scanned >= s.maxScan } -// warnIfTruncated logs when a scan hit its safety cap, so one-shot output signals +// warnIfTruncated warns when a scan hit its safety cap, so the output signals // its results may be incomplete. func warnIfTruncated(ctx context.Context, f *runFetcher) { - if f.strategy.truncated() { - log.Warnf(ctx, "air list: stopped after scanning %d runs; results may be incomplete", maxListScan) + s, ok := f.strategy.(*jobsScanStrategy) + if !ok || !s.truncated() { + return } + log.Warnf(ctx, "air list: stopped after scanning %d runs; results may be incomplete. Narrow the search with --filter or a smaller --limit", s.scanned) } // setMLflowLinks fills in each row's MLflow link, label, and experiment URL - diff --git a/cmd/air/list_index_test.go b/cmd/air/list_index_test.go index e6ed2d0df41..0d4f37887c0 100644 --- a/cmd/air/list_index_test.go +++ b/cmd/air/list_index_test.go @@ -1,14 +1,21 @@ package aircmd import ( + "bytes" + "context" "encoding/json" "fmt" + "log/slog" "net/http" "net/http/httptest" "strconv" "sync/atomic" "testing" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -172,6 +179,8 @@ func TestNewListStrategyGate(t *testing.T) { {"all-status self → index", listQuery{currentUser: selfUser, limit: 5}, true}, {"all-status all-users → scan", listQuery{allUsers: true}, false}, {"all-status other user → scan", listQuery{userFilter: "other@example.com"}, false}, + {"all-status self filter, other case → index", listQuery{userFilter: "Me@Example.com", selfUser: "me@example.com", limit: 5}, true}, + {"all-status filter, current user unknown → scan", listQuery{userFilter: "me@example.com"}, false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -192,7 +201,7 @@ func TestNewListStrategyGate(t *testing.T) { } func TestNewListStrategyFallsBackWhenIndexFails(t *testing.T) { - // Index 500 must silently fall back to the Jobs scan, not fail the command. + // Index 500 must fall back to the Jobs scan, not fail the command. srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == aiTrainingWorkflowsPath { w.WriteHeader(http.StatusInternalServerError) @@ -236,3 +245,121 @@ func TestIndexStrategySkipsUserLookup(t *testing.T) { require.Len(t, rows, 1) assert.Zero(t, lookups.Load()) } + +// captureWarnings routes warnings logged through ctx into a buffer. +func captureWarnings(ctx context.Context) (context.Context, *bytes.Buffer) { + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelWarn})) + return log.NewContext(ctx, logger), &buf +} + +// endlessNonAirRunsPage is a full runs/list page of non-AIR runs that always +// points at another page, standing in for a deep run history. +func endlessNonAirRunsPage(t *testing.T) string { + runs := make([]jobs.BaseRun, jobsPageLimit) + for i := range runs { + runs[i] = jobs.BaseRun{RunId: int64(i + 1), CreatorUserName: "me@example.com", Tasks: []jobs.RunTask{{}}} + } + return runsListBody(t, "more", runs...) +} + +func TestFallbackScanWarnsAndIsBounded(t *testing.T) { + var listCalls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case aiTrainingWorkflowsPath: + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"message":"boom"}`)) + case "/api/2.2/jobs/runs/list": + listCalls.Add(1) + assert.Empty(t, r.URL.Query().Get("active_only")) + _, _ = fmt.Fprint(w, endlessNonAirRunsPage(t)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + t.Setenv("DATABRICKS_CACHE_ENABLED", "false") + + var stdout bytes.Buffer + ctx, warnings := captureWarnings(t.Context()) + ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputJSON, nil, &stdout, &stdout, "", "")) + f := newRunFetcher(ctx, newTestWorkspaceClient(t, srv.URL), listQuery{ + userFilter: "me@example.com", selfUser: "me@example.com", limit: 10, + }) + rows, err := f.next(10) + require.NoError(t, err) + assert.Empty(t, rows) + warnIfTruncated(ctx, f) + + // The fallback stops at its own cap instead of maxListScan. + assert.Equal(t, int32(maxFallbackListScan/jobsPageLimit), listCalls.Load()) + assert.Contains(t, warnings.String(), "run index is unavailable") + assert.Contains(t, warnings.String(), "falling back to a slower scan of up to 250 recent job runs") + assert.Contains(t, warnings.String(), "stopped after scanning 250 runs; results may be incomplete") + assert.Empty(t, stdout.String()) +} + +func TestRegularScanKeepsFullCap(t *testing.T) { + var listCalls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/2.2/jobs/runs/list" { + listCalls.Add(1) + _, _ = fmt.Fprint(w, endlessNonAirRunsPage(t)) + return + } + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + + ctx, warnings := captureWarnings(t.Context()) + f := newRunFetcher(ctx, newTestWorkspaceClient(t, srv.URL), listQuery{allUsers: true, limit: 10}) + _, err := f.next(10) + require.NoError(t, err) + warnIfTruncated(ctx, f) + + assert.Equal(t, int32(maxListScan/jobsPageLimit), listCalls.Load()) + assert.NotContains(t, warnings.String(), "run index is unavailable") + assert.Contains(t, warnings.String(), "stopped after scanning 2000 runs; results may be incomplete") +} + +// TestListSelfUserFilterUsesIndex pins that --all-status --filter user= is served by the index, like the unfiltered self list, rather than a +// full-history Jobs scan. +func TestListSelfUserFilterUsesIndex(t *testing.T) { + var indexHits, listHits atomic.Int32 + run := indexRun(7, 3000_000) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/2.0/preview/scim/v2/Me": + _, _ = w.Write([]byte(`{"userName":"me@example.com"}`)) + case aiTrainingWorkflowsPath: + indexHits.Add(1) + _, _ = w.Write([]byte(`{"training_workflows":[{"job_run_id":"7","submit_time":{"seconds":3000}}]}`)) + case "/api/2.2/jobs/runs/get": + _ = json.NewEncoder(w).Encode(run) + case "/api/2.2/jobs/runs/list": + listHits.Add(1) + _, _ = w.Write([]byte(`{}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + t.Setenv("DATABRICKS_CACHE_ENABLED", "false") + + var stdout bytes.Buffer + ctx, warnings := captureWarnings(t.Context()) + ctx = cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srv.URL)) + ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputJSON, nil, &stdout, &stdout, "", "")) + cmd := withOutput(newListCommand(), flags.OutputJSON) + cmd.SetContext(ctx) + require.NoError(t, cmd.Flags().Set("all-status", "true")) + require.NoError(t, cmd.Flags().Set("filter", "user=me@example.com")) + + require.NoError(t, cmd.RunE(cmd, nil)) + assert.Equal(t, int32(1), indexHits.Load()) + assert.Equal(t, int32(0), listHits.Load()) + assert.Contains(t, stdout.String(), `"run_id": "7"`) + assert.Empty(t, warnings.String()) +} diff --git a/cmd/air/list_tui.go b/cmd/air/list_tui.go index af5951bca44..3245eee8f90 100644 --- a/cmd/air/list_tui.go +++ b/cmd/air/list_tui.go @@ -46,7 +46,11 @@ func renderListText(cmd *cobra.Command, f *runFetcher, limit int, activeOnly boo tea.WithInput(cmd.InOrStdin()), tea.WithOutput(out), ).Run() - return err + if err != nil { + return err + } + warnIfTruncated(ctx, f) + return nil } rows, err := f.next(limit) From ba33c5ce2f11d28490a3a4f8cc621ebb6dec97b0 Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Fri, 13 Nov 2026 23:47:21 +0000 Subject: [PATCH 2/2] air: keep the list truncation warning on the strategy interface and update the fallback golden Co-authored-by: Isaac --- .nextchanges/air/list-index-fallback.md | 2 +- acceptance/air/list-index-fallback/output.txt | 7 ++++++- cmd/air/list.go | 7 ++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.nextchanges/air/list-index-fallback.md b/.nextchanges/air/list-index-fallback.md index 127dd497b7d..ab5fe380913 100644 --- a/.nextchanges/air/list-index-fallback.md +++ b/.nextchanges/air/list-index-fallback.md @@ -1 +1 @@ -* `databricks air list --all-status` now warns when the AI Runtime run index is unavailable and it falls back to scanning Jobs runs. That fallback scan is capped at 250 runs, and a stuck index request fails after 15 seconds instead of retrying for minutes. `--all-status --filter user=` now uses the run index instead of scanning the workspace's full run history. +* `databricks air list --all-status` now warns when the AI Runtime run index is unavailable and it falls back to scanning Jobs runs. That fallback scan is capped at 250 runs, and a stuck index request fails after 15 seconds instead of retrying for minutes. `--all-status --filter user=` now uses the run index instead of scanning the workspace's full run history. ([#7619](https://github.com/databricks/cli/pull/7619)) diff --git a/acceptance/air/list-index-fallback/output.txt b/acceptance/air/list-index-fallback/output.txt index 9460c16bc81..8cc9da7c7fd 100644 --- a/acceptance/air/list-index-fallback/output.txt +++ b/acceptance/air/list-index-fallback/output.txt @@ -18,7 +18,12 @@ Warn: air list: the AI Runtime run index is unavailable (failed to list training "user": "[USERNAME]", "status": "SUCCESS", "started_at": "[TIMESTAMP]", - "is_sweep": false + "is_sweep": false, + "experiment_name": "qwen-train", + "duration_seconds": 12, + "accelerator_type": "GPU_8xH100", + "num_accelerators": 8, + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/334747067049496?o=[WORKSPACE_ID]" } ] } diff --git a/cmd/air/list.go b/cmd/air/list.go index b9d991c9bea..a87f05e68b2 100644 --- a/cmd/air/list.go +++ b/cmd/air/list.go @@ -401,8 +401,13 @@ func (s *jobsScanStrategy) truncated() bool { // warnIfTruncated warns when a scan hit its safety cap, so the output signals // its results may be incomplete. func warnIfTruncated(ctx context.Context, f *runFetcher) { + if !f.strategy.truncated() { + return + } + // Only the scan strategy can report truncated, and it carries the scanned + // count for the warning. s, ok := f.strategy.(*jobsScanStrategy) - if !ok || !s.truncated() { + if !ok { return } log.Warnf(ctx, "air list: stopped after scanning %d runs; results may be incomplete. Narrow the search with --filter or a smaller --limit", s.scanned)