Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/get-max-retries.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `air get` and the `air list` info pane no longer show `Max Retries 0` when the Jobs API leaves the task's retry limit out. For a run submitted with `air run`, the limit now comes from the run's uploaded config, shown as `3 (CLI default)` when the config doesn't set `max_retries`; it shows `N/A` when the limit can't be known (job runs, multi-container runs, legacy runs with no explicit value, or an unreadable config). ([#7623](https://github.com/databricks/cli/pull/7623))
36 changes: 35 additions & 1 deletion acceptance/air/get-ai-runtime/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
│ Status ● SUCCESS │
│ Submitted 2023-11-14 22:13 UTC │
│ Retries 0 │
│ Max Retries 3 │
│ Max Retries 3 (CLI default) │
│ Duration 12s │
│ Experiment my-exp │
│ MLflow Run my-run │
Expand Down Expand Up @@ -54,3 +54,37 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1
"est_percent_complete": null
}
}

=== get (text, max_retries set in the config)

>>> [CLI] air get 123

╭─ Configuration ────────────────────────────────────────────────╮
│ │
│ experiment_name: my-exp │
│ compute: │
│ accelerator_type: a10 │
│ num_accelerators: 1 │
│ max_retries: 1 │
│ command: echo ok │
│ │
╰────────────────────────────────────────────────────────────────╯

╭─ Metadata ─────────────────────────────────────────────────────╮
│ │
│ Run ID 123 │
│ Status ● SUCCESS │
│ Submitted 2023-11-14 22:13 UTC │
│ Retries 0 │
│ Max Retries 1 │
│ Duration 12s │
│ Experiment my-exp │
│ MLflow Run my-run │
│ User user@example.com │
│ Accelerators 1x A10 │
│ Environment 4 │
│ │
╰────────────────────────────────────────────────────────────────╯

Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]
MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1
7 changes: 7 additions & 0 deletions acceptance/air/get-ai-runtime/script
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,10 @@ trace $CLI air get 123

title "get (json)"
trace $CLI air get 123 -o json

# The fake runs/get, like the real one, has no max_retries on the task, so the
# limit comes from the uploaded config: the CLI default above, the set value here.
$CLI workspace import "/Workspace/Users/user@example.com/.air/cli_launch/my-exp/my-exp_abc/training_config.yaml" --file training_config_max_retries.yaml --format AUTO --overwrite &> LOG.import-max-retries

title "get (text, max_retries set in the config)"
trace $CLI air get 123
3 changes: 2 additions & 1 deletion acceptance/air/get-ai-runtime/test.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ Response.Body = ''
# so the Environment cell instead comes from the run's environments[].spec (keyed
# by the task's environment_key), which the typed SDK Run drops — `air get` reads
# it with a raw request, so this route is hit twice (typed GetRun + that request).
# Like the real runs/get, the task carries no max_retries.
[[Server]]
Pattern = "GET /api/2.2/jobs/runs/get"
Response.Body = '''
{
"run_id": 123,
"run_type": "SUBMIT_RUN",
"run_page_url": "https://my-workspace.cloud.databricks.test/jobs/runs/123",
"creator_user_name": "user@example.com",
"start_time": 1700000000000,
Expand All @@ -30,7 +32,6 @@ Response.Body = '''
"task_key": "train",
"run_id": 456,
"attempt_number": 0,
"max_retries": 3,
"environment_key": "default",
"ai_runtime_task": {
"experiment": "my-exp",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
experiment_name: my-exp
compute:
accelerator_type: a10
num_accelerators: 1
max_retries: 1
command: echo ok
2 changes: 1 addition & 1 deletion acceptance/air/get/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
│ Status ● SUCCESS │
│ Submitted 2023-11-14 22:13 UTC │
│ Retries 0 │
│ Max Retries 3 │
│ Max Retries N/A │
│ Duration 12s │
│ Experiment my-exp │
│ MLflow Run my-run │
Expand Down
6 changes: 4 additions & 2 deletions acceptance/air/get/test.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,15 @@
Pattern = "HEAD /"
Response.Body = ''

# A single GenAI-compute run with an experiment, GPUs, and a creator.
# A single GenAI-compute run with an experiment, GPUs, and a creator. Like the
# real runs/get, the task carries no max_retries, and its yaml_parameters leave
# max_retries unset, so the limit is unknown.
[[Server]]
Pattern = "GET /api/2.2/jobs/runs/get"
Response.Body = '''
{
"run_id": 123,
"run_type": "SUBMIT_RUN",
"run_page_url": "https://my-workspace.cloud.databricks.test/jobs/runs/123",
"creator_user_name": "user@example.com",
"start_time": 1700000000000,
Expand All @@ -20,7 +23,6 @@ Response.Body = '''
"task_key": "train",
"run_id": 456,
"attempt_number": 0,
"max_retries": 3,
"gen_ai_compute_task": {
"mlflow_experiment_name": "/Users/user@example.com/my-exp",
"compute": {"gpu_type": "GPU_1xA10", "num_gpus": 1},
Expand Down
107 changes: 99 additions & 8 deletions cmd/air/format.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"io"
"math"
"slices"
"strconv"
"strings"
"time"
Expand Down Expand Up @@ -359,15 +360,105 @@ func gpuDisplayName(gpuType string) string {
return gpuType
}

// maxRetries returns the configured retry limit for the run's latest task as a
// display string: "unlimited" for the backend's -1, otherwise the count.
func maxRetries(run *jobs.Run) string {
// maxRetriesSource says where a run's retry limit was read from.
type maxRetriesSource string

const (
// maxRetriesReported: runs/get returned max_retries on the task.
maxRetriesReported maxRetriesSource = "reported"
// maxRetriesConfigured: the run's uploaded config sets max_retries.
maxRetriesConfigured maxRetriesSource = "configured"
// maxRetriesCLIDefault: the config leaves max_retries unset, and the CLI
// submitted its default.
maxRetriesCLIDefault maxRetriesSource = "cli_default"
)

// runMaxRetries is a run's retry limit and where it came from. Limit -1 means
// unlimited.
type runMaxRetries struct {
Limit int
Source maxRetriesSource
}

// resolveMaxRetries returns the retry limit of the run's latest task, or nil
// when it can't be known. configRaw is the run's training config as uploaded
// (nil when it was not found or could not be downloaded).
//
// runs/get omits max_retries on task runs, and the SDK decodes the missing field
// as 0, so a value is trusted only when the response actually carried the key.
// Otherwise the limit is read from the config, and only for one-time submit
// runs: a job run (for example a bundle deployed with `air convert-to-dabs`)
// takes its retries from the job settings, which the config doesn't track. When
// the config leaves the key unset, the CLI default is assumed only for
// ai_runtime_task runs, because every CLI that submits that task type sends 3
// (runsubmit.go always sends max_retries). Legacy gen_ai_compute_task runs may
// predate the Python CLI's switch from 0 to 3, so their unset limit is unknown.
func resolveMaxRetries(run *jobs.Run, configRaw []byte) *runMaxRetries {
if n, ok := reportedMaxRetries(run); ok {
return &runMaxRetries{Limit: n, Source: maxRetriesReported}
}
if run.RunType != jobs.RunTypeSubmitRun {
return nil
}
n, set, ok := configMaxRetries(configRaw)
switch {
case !ok:
return nil
case set:
return &runMaxRetries{Limit: n, Source: maxRetriesConfigured}
case aiRuntimeTaskOf(run) != nil:
return &runMaxRetries{Limit: defaultMaxRetries, Source: maxRetriesCLIDefault}
default:
return nil
}
}

// maxRetriesDisplay renders a retry limit for the text view: "N/A" when it is
// unknown, "unlimited" for -1, and a "(CLI default)" note when the config left
// it unset.
func maxRetriesDisplay(m *runMaxRetries) string {
if m == nil {
return na
}
s := strconv.Itoa(m.Limit)
if m.Limit < 0 {
s = "unlimited"
}
if m.Source == maxRetriesCLIDefault {
s += " (CLI default)"
}
return s
}

// reportedMaxRetries returns the latest task's max_retries if the runs/get
// response carried the key. The SDK records a decoded zero value in
// ForceSendFields, which tells an explicit 0 apart from a missing field.
func reportedMaxRetries(run *jobs.Run) (int, bool) {
if len(run.Tasks) == 0 {
return "0"
return 0, false
}
t := run.Tasks[len(run.Tasks)-1]
if t.MaxRetries == 0 && !slices.Contains(t.ForceSendFields, "MaxRetries") {
return 0, false
}
return t.MaxRetries, true
}

// configMaxRetries reads max_retries from a run's config YAML: the value, whether
// the key is set, and whether the config parsed at all. It decodes only that
// key so configs written by other CLI versions still parse.
func configMaxRetries(raw []byte) (n int, set, ok bool) {
if len(raw) == 0 {
return 0, false, false
}
var cfg struct {
MaxRetries *int `yaml:"max_retries"`
}
if err := yaml.Unmarshal(raw, &cfg); err != nil {
return 0, false, false
}
n := run.Tasks[len(run.Tasks)-1].MaxRetries
if n < 0 {
return "unlimited"
if cfg.MaxRetries == nil {
return 0, false, true
}
return strconv.Itoa(n)
return *cfg.MaxRetries, true, true
}
92 changes: 92 additions & 0 deletions cmd/air/format_test.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
package aircmd

import (
"encoding/json"
"io"
"strconv"
"testing"

"github.com/databricks/cli/libs/cmdio"
Expand Down Expand Up @@ -306,3 +308,93 @@ func TestTerminationReason(t *testing.T) {
assert.Nil(t, terminationReason(&jobs.Run{State: &jobs.RunState{ResultState: jobs.RunResultStateSuccess, StateMessage: "done"}}))
assert.Nil(t, terminationReason(&jobs.Run{State: &jobs.RunState{ResultState: jobs.RunResultStateCanceled, StateMessage: "cancelled"}}))
}

func TestMaxRetries(t *testing.T) {
submit := func(task jobs.RunTask) *jobs.Run {
return &jobs.Run{RunType: jobs.RunTypeSubmitRun, Tasks: []jobs.RunTask{task}}
}
aiRuntime := jobs.RunTask{AiRuntimeTask: &jobs.AiRuntimeTask{}}
reported := func(n int) jobs.RunTask {
return jobs.RunTask{MaxRetries: n, AiRuntimeTask: &jobs.AiRuntimeTask{}, ForceSendFields: []string{"MaxRetries"}}
}
legacy := jobs.RunTask{GenAiComputeTask: &jobs.GenAiComputeTask{}}
jobRun := &jobs.Run{RunType: jobs.RunTypeJobRun, Tasks: []jobs.RunTask{aiRuntime}}
tests := []struct {
name string
run *jobs.Run
config string
want string
}{
{"submit, config sets 1", submit(aiRuntime), "max_retries: 1\n", "1"},
{"submit, config sets 0", submit(aiRuntime), "max_retries: 0\n", "0"},
{"submit, config sets unlimited", submit(aiRuntime), "max_retries: -1\n", "unlimited"},
{"submit, config unset uses the CLI default", submit(aiRuntime), "experiment_name: e\n", "3 (CLI default)"},
{"submit, no config", submit(aiRuntime), "", "N/A"},
{"submit, unparsable config", submit(aiRuntime), "[not: a map", "N/A"},
{"legacy submit, config unset is unknown", submit(legacy), "experiment_name: e\n", "N/A"},
{"legacy submit, config sets 2", submit(legacy), "max_retries: 2\n", "2"},
{"job run ignores the config", jobRun, "max_retries: 1\n", "N/A"},
{"job run, config unset", jobRun, "experiment_name: e\n", "N/A"},
{"run type unknown", &jobs.Run{Tasks: []jobs.RunTask{aiRuntime}}, "max_retries: 1\n", "N/A"},
{"server reports 0 over the config", submit(reported(0)), "max_retries: 2\n", "0"},
{"server reports 5 on a job run", &jobs.Run{RunType: jobs.RunTypeJobRun, Tasks: []jobs.RunTask{reported(5)}}, "", "5"},
{"server reports unlimited", submit(reported(-1)), "", "unlimited"},
{"no tasks", &jobs.Run{RunType: jobs.RunTypeSubmitRun}, "", "N/A"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var raw []byte
if tt.config != "" {
raw = []byte(tt.config)
}
assert.Equal(t, tt.want, maxRetriesDisplay(resolveMaxRetries(tt.run, raw)))
})
}
}

func TestResolveMaxRetriesSource(t *testing.T) {
run := &jobs.Run{RunType: jobs.RunTypeSubmitRun, Tasks: []jobs.RunTask{{AiRuntimeTask: &jobs.AiRuntimeTask{}}}}
assert.Equal(t, &runMaxRetries{Limit: 1, Source: maxRetriesConfigured}, resolveMaxRetries(run, []byte("max_retries: 1\n")))
assert.Equal(t, &runMaxRetries{Limit: 3, Source: maxRetriesCLIDefault}, resolveMaxRetries(run, []byte("command: x\n")))
run.Tasks[0].MaxRetries = 2
assert.Equal(t, &runMaxRetries{Limit: 2, Source: maxRetriesReported}, resolveMaxRetries(run, nil))
}

// The runs/get bodies below are trimmed copies of real workspace responses:
// task runs carry no max_retries key at all.
func TestMaxRetriesFromDecodedRunGet(t *testing.T) {
decode := func(body string) *jobs.Run {
var run jobs.Run
require.NoError(t, json.Unmarshal([]byte(body), &run))
return &run
}
display := func(run *jobs.Run, config []byte) string {
return maxRetriesDisplay(resolveMaxRetries(run, config))
}
// Run 111111111111111: submitted with max_retries 1, failed, retried once.
aiRuntime := decode(`{"run_id": 111111111111111, "run_type": "SUBMIT_RUN", "tasks": [
{"task_key": "t", "run_id": 222222222222222, "attempt_number": 0, "ai_runtime_task": {"experiment": "e"}},
{"task_key": "t", "run_id": 333333333333333, "attempt_number": 1, "ai_runtime_task": {"experiment": "e"}}]}`)
assert.Equal(t, "1", display(aiRuntime, []byte("experiment_name: e\nmax_retries: 1\n")))
assert.Equal(t, "3 (CLI default)", display(aiRuntime, []byte("experiment_name: e\n")))

// A multi-container run has no deployment-level command_path, so the config
// path is unknown and nothing is downloaded.
multi := decode(`{"run_id": 1, "run_type": "SUBMIT_RUN", "tasks": [{"task_key": "t", "run_id": 2,
"ai_runtime_task": {"experiment": "e", "deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}}]}}]}`)
var data getData
enrichFromAiRuntimeTask(multi, &data)
assert.Empty(t, data.TrainingConfigPath)
assert.Equal(t, "N/A", display(multi, nil))

// A legacy run as the Python CLI submitted it: yaml_parameters is the user's
// raw YAML, and max_retries is sent only when > 0.
legacyYAML := "experiment_name: my-exp\ncompute:\n accelerator_type: a10\n num_accelerators: 1\ncommand: python train.py\n"
legacy := decode(`{"run_id": 1, "run_type": "SUBMIT_RUN", "tasks": [{"task_key": "train", "run_id": 2, "attempt_number": 0,
"gen_ai_compute_task": {"mlflow_experiment_name": "/Users/u/my-exp", "yaml_parameters": ` + strconv.Quote(legacyYAML) + `}}]}`)
_, raw := configYAML(t.Context(), nil, genAIComputeTask(legacy))
assert.Equal(t, "N/A", display(legacy, raw))

explicitZero := decode(`{"run_type": "SUBMIT_RUN", "tasks": [{"max_retries": 0, "ai_runtime_task": {}}]}`)
assert.Equal(t, "0", display(explicitZero, []byte("experiment_name: e\n")))
}
2 changes: 0 additions & 2 deletions cmd/air/get.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@ type getData struct {
UserDisplay string `json:"-"`
AcceleratorsDisplay string `json:"-"`
EnvironmentDisplay string `json:"-"`
MaxRetriesDisplay string `json:"-"`
// ProgressDisplay is the pre-rendered "Progress" cell ("45% · ~2h 15m left"),
// set only for a running run with an estimable remaining time.
ProgressDisplay string `json:"-"`
Expand Down Expand Up @@ -292,6 +291,5 @@ func buildGetData(run *jobs.Run) getData {
// version needs a raw GetRun read (the typed SDK Run drops it), so it is not
// filled here alongside the fields read straight off the run.
data.EnvironmentDisplay = na
data.MaxRetriesDisplay = maxRetries(run)
return data
}
3 changes: 3 additions & 0 deletions cmd/air/list_detail.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ func runDetailText(ctx context.Context, w *databricks.WorkspaceClient, runID int

data := buildGetData(run)
data.DashboardURL = dashboardURL(w.Config.Host, runID, workspaceIDForRunLink(ctx, w))
if findForEachTask(run) == nil && genAIComputeTask(run) == nil {
enrichFromAiRuntimeTask(run, &data)
}
ids := mlflowIDs(ctx, w, run)
if ids != nil {
url := mlflowLogsURL(w.Config.Host, ids)
Expand Down
Loading
Loading