Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/list-index-fallback.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `databricks air list --all-status` now warns when the AI Runtime run index is unavailable and it falls back to scanning Jobs runs. That fallback scan is capped at 250 runs, and a stuck index request fails after 15 seconds instead of retrying for minutes. `--all-status --filter user=<your email>` now uses the run index instead of scanning the workspace's full run history. ([#7619](https://github.com/databricks/cli/pull/7619))
1 change: 1 addition & 0 deletions acceptance/air/list-index-fallback/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
30 changes: 30 additions & 0 deletions acceptance/air/list-index-fallback/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@

=== list --all-status falls back to a bounded Jobs scan when the index fails (text)

>>> [CLI] air list --all-status
Warn: air list: the AI Runtime run index is unavailable (failed to list training workflows: index unavailable); falling back to a slower scan of up to 250 recent job runs
Run ID Experiment Status Started Duration Progress MLflow User Accelerators
334747067049496 qwen-train ● SUCCESS [TIMESTAMP] 12s - qwen-train-001 [USERNAME] 8x H100

=== list --all-status fallback (json): the warning stays on stderr
{
"v": 1,
"ts": "[TIMESTAMP]",
"data": {
"runs": [
{
"run_id": "334747067049496",
"run_name": "qwen-train",
"user": "[USERNAME]",
"status": "SUCCESS",
"started_at": "[TIMESTAMP]",
"is_sweep": false,
"experiment_name": "qwen-train",
"duration_seconds": 12,
"accelerator_type": "GPU_8xH100",
"num_accelerators": 8,
"dashboard_url": "[DATABRICKS_URL]/jobs/runs/334747067049496?o=[WORKSPACE_ID]"
}
]
}
}
5 changes: 5 additions & 0 deletions acceptance/air/list-index-fallback/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
title "list --all-status falls back to a bounded Jobs scan when the index fails (text)"
trace $CLI air list --all-status

title "list --all-status fallback (json): the warning stays on stderr"
trace $CLI air list --all-status -o json 2>/dev/null
49 changes: 49 additions & 0 deletions acceptance/air/list-index-fallback/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
# Disable the on-disk run cache so --all-status output is deterministic across runs.
Env.DATABRICKS_CACHE_ENABLED = "false"

[[Server]]
Pattern = "HEAD /"
Response.Body = ''

# The AiTrainingService index is unavailable, so --all-status falls back to
# scanning Jobs runs/list.
[[Server]]
Pattern = "GET /api/2.0/ai-training/workflows"
Response.StatusCode = 500
Response.Body = '{"error_code": "INTERNAL_ERROR", "message": "index unavailable"}'

[[Server]]
Pattern = "GET /api/2.2/jobs/runs/list"
Response.Body = '''
{
"runs": [
{
"run_id": 334747067049496,
"run_name": "qwen-train",
"creator_user_name": "tester@databricks.com",
"start_time": 1717608759000,
"end_time": 1717608771000,
"state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"},
"tasks": [{
"run_id": 334747067049497,
"ai_runtime_task": {
"experiment": "/Users/tester@databricks.com/qwen-train",
"deployments": [{"compute": {"accelerator_type": "GPU_8xH100", "accelerator_count": 8}}]
}
}]
}
]
}
'''

[[Server]]
Pattern = "GET /api/2.2/jobs/runs/get-output"
Response.Body = '''
{"ai_runtime_task_output": {"mlflow_experiment_id": "exp1", "mlflow_run_id": "run1"}}
'''

[[Server]]
Pattern = "GET /api/2.0/mlflow/runs/get"
Response.Body = '''
{"run": {"info": {"run_name": "qwen-train-001"}}}
'''
24 changes: 23 additions & 1 deletion cmd/air/aitraining.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import (
"github.com/databricks/cli/libs/auth"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/client"
"github.com/databricks/databricks-sdk-go/config"
"github.com/databricks/databricks-sdk-go/httpclient"
)

// aiTrainingWorkflowsPath is the AiTrainingService index of the caller's own AIR
Expand Down Expand Up @@ -71,11 +73,31 @@ type aiTrainingWorkflowsResponse struct {
NextPageToken string `json:"next_page_token"`
}

// The index is an optimization with a Jobs-scan fallback, so a slow or hung
// index page should fail fast rather than ride the SDK defaults (a 60s request
// timeout retried for up to 5 minutes). These bound each page request.
const (
aiTrainingIndexRequestTimeout = 15 * time.Second
aiTrainingIndexRetryTimeout = 15 * time.Second
)

// newAiTrainingIndexClient returns an API client for the index with the shorter
// per-request timeout and retry window. It leaves w.Config untouched.
func newAiTrainingIndexClient(w *databricks.WorkspaceClient) (*client.DatabricksClient, error) {
clientCfg, err := config.HTTPClientConfigFromConfig(w.Config)
if err != nil {
return nil, err
}
clientCfg.HTTPTimeout = aiTrainingIndexRequestTimeout
clientCfg.RetryTimeout = aiTrainingIndexRetryTimeout
return client.NewWithClient(w.Config, httpclient.NewApiClient(clientCfg))
}

// listAiTrainingWorkflows pages the index and returns every workflow ref the
// caller owns. Pagination stops at the end or when a page token repeats, which
// guards against a stuck or cycling cursor without an arbitrary page cap.
func listAiTrainingWorkflows(ctx context.Context, w *databricks.WorkspaceClient, activeOnly bool) ([]workflowRef, error) {
apiClient, err := client.New(w.Config)
apiClient, err := newAiTrainingIndexClient(w)
if err != nil {
return nil, fmt.Errorf("failed to create API client: %w", err)
}
Expand Down
72 changes: 57 additions & 15 deletions cmd/air/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package aircmd
import (
"context"
"fmt"
"strings"
"sync"

"github.com/databricks/cli/cmd/root"
Expand All @@ -21,6 +22,11 @@ import (
// work on a workspace with a large run history.
const maxListScan = 2000

// maxFallbackListScan bounds the Jobs scan that stands in for an unavailable
// AiTrainingService index. That scan can't use active_only, so it walks the full
// run history, which costs several seconds per page on a busy workspace.
const maxFallbackListScan = 250

// jobsPageLimit is the per-request page size for runs/list; enrichConcurrency
// bounds the parallel MLflow lookups.
const (
Expand Down Expand Up @@ -80,6 +86,10 @@ type listQuery struct {
// currentUser resolves the caller's user name for a self-scoped list. It is
// nil when an explicit user= filter or --all-users is set.
currentUser func() (string, error)
// selfUser is the caller's user name when it was resolved up front, despite
// an explicit user= filter (--all-status only). The strategy compares it
// with userFilter to tell a self filter from a filter for another user.
selfUser string
filters listFilters
fetchMLflow bool
limit int
Expand Down Expand Up @@ -145,17 +155,34 @@ Filters:
// An explicit user= filter wins; otherwise default to the current user
// unless --all-users is set. The user is resolved only if a strategy
// needs it: the index is already scoped to the caller, and runs/list has
// no creator param, so the scan matches the creator itself.
// no creator param, so the scan matches the creator itself. With
// --all-status the current user is also resolved for an explicit user=
// filter, so filtering on your own email uses the index instead of a
// full-history scan.
var currentUser func() (string, error)
if f.User == "" && !allUsers {
currentUser = sync.OnceValues(func() (string, error) { return currentUserEmail(ctx, w) })
var selfUser string
if !allUsers && (f.User == "" || allStatus) {
if f.User == "" {
currentUser = sync.OnceValues(func() (string, error) { return currentUserEmail(ctx, w) })
} else {
// Resolve the caller up front so the strategy can tell a self
// filter from a filter for another user. A lookup failure keeps
// the scan, which matches the creator itself.
user, err := currentUserEmail(ctx, w)
if err != nil {
log.Debugf(ctx, "air list: could not resolve current user, scanning Jobs runs for user=%s: %v", f.User, err)
} else {
selfUser = user
}
}
}

fetcher := newRunFetcher(ctx, w, listQuery{
activeOnly: !allStatus,
allUsers: allUsers,
userFilter: f.User,
currentUser: currentUser,
selfUser: selfUser,
filters: f,
fetchMLflow: root.OutputType(cmd) == flags.OutputText,
limit: limit,
Expand Down Expand Up @@ -234,17 +261,22 @@ func newRunFetcher(ctx context.Context, w *databricks.WorkspaceClient, q listQue
// newListStrategy picks the fetch source. The AiTrainingService index serves only
// the caller's own runs, so it's used for an all-status self-scoped list; if the
// index load fails (e.g. endpoint unavailable in this workspace), we fall back to
// the Jobs scan so the command still returns. Everything else — the default
// a bounded Jobs scan so the command still returns. Everything else — the default
// active list, --all-users, and --all-status for another user — uses the scan.
func newListStrategy(ctx context.Context, w *databricks.WorkspaceClient, q listQuery) listStrategy {
useIndex := !q.activeOnly && q.currentUser != nil
// The index serves only the caller's own runs: a list with no user filter
// (the scan would resolve the caller itself), or an explicit user= filter
// that matches the caller case-insensitively (resolved up front under
// --all-status).
isSelf := q.userFilter == "" || (q.selfUser != "" && strings.EqualFold(q.userFilter, q.selfUser))
useIndex := !q.activeOnly && !q.allUsers && isSelf
if !useIndex {
return newJobsScanStrategy(ctx, w, q)
return newJobsScanStrategy(ctx, w, q, maxListScan)
}
idx := newIndexStrategy(ctx, w, q, q.limit)
if err := idx.load(); err != nil {
log.Debugf(ctx, "air list: AiTrainingService index unavailable, falling back to Jobs scan: %v", err)
return newJobsScanStrategy(ctx, w, q)
log.Warnf(ctx, "air list: the AI Runtime run index is unavailable (%v); falling back to a slower scan of up to %d recent job runs", err, maxFallbackListScan)
return newJobsScanStrategy(ctx, w, q, maxFallbackListScan)
}
return idx
}
Expand Down Expand Up @@ -297,10 +329,12 @@ type jobsScanStrategy struct {
host string
workspaceID int64

// maxScan caps how many runs this scan inspects; see maxListScan.
maxScan int
scanned int
}

func newJobsScanStrategy(ctx context.Context, w *databricks.WorkspaceClient, q listQuery) *jobsScanStrategy {
func newJobsScanStrategy(ctx context.Context, w *databricks.WorkspaceClient, q listQuery, maxScan int) *jobsScanStrategy {
req := jobs.ListRunsRequest{
RunType: jobs.RunTypeSubmitRun,
ExpandTasks: true,
Expand All @@ -321,12 +355,13 @@ func newJobsScanStrategy(ctx context.Context, w *databricks.WorkspaceClient, q l
filters: q.filters,
host: w.Config.Host,
workspaceID: q.workspaceID,
maxScan: maxScan,
}
}

func (s *jobsScanStrategy) next(want int) ([]listedRun, error) {
var entries []listedRun
for len(entries) < want && s.scanned < maxListScan && s.iter.HasNext(s.ctx) {
for len(entries) < want && s.scanned < s.maxScan && s.iter.HasNext(s.ctx) {
base, err := s.iter.Next(s.ctx)
if err != nil {
return nil, fmt.Errorf("failed to list runs: %w", err)
Expand Down Expand Up @@ -356,19 +391,26 @@ func (s *jobsScanStrategy) next(want int) ([]listedRun, error) {
}

func (s *jobsScanStrategy) done() bool {
return s.scanned >= maxListScan || !s.iter.HasNext(s.ctx)
return s.scanned >= s.maxScan || !s.iter.HasNext(s.ctx)
}

func (s *jobsScanStrategy) truncated() bool {
return s.scanned >= maxListScan
return s.scanned >= s.maxScan
}

// warnIfTruncated logs when a scan hit its safety cap, so one-shot output signals
// warnIfTruncated warns when a scan hit its safety cap, so the output signals
// its results may be incomplete.
func warnIfTruncated(ctx context.Context, f *runFetcher) {
if f.strategy.truncated() {
log.Warnf(ctx, "air list: stopped after scanning %d runs; results may be incomplete", maxListScan)
if !f.strategy.truncated() {
return
}
// Only the scan strategy can report truncated, and it carries the scanned
// count for the warning.
s, ok := f.strategy.(*jobsScanStrategy)
if !ok {
return
}
log.Warnf(ctx, "air list: stopped after scanning %d runs; results may be incomplete. Narrow the search with --filter or a smaller --limit", s.scanned)
}

// setMLflowLinks fills in each row's MLflow link, label, and experiment URL -
Expand Down
Loading
Loading