diff --git a/.nextchanges/air/get-max-retries.md b/.nextchanges/air/get-max-retries.md new file mode 100644 index 00000000000..64792857e5c --- /dev/null +++ b/.nextchanges/air/get-max-retries.md @@ -0,0 +1 @@ +* `air get` and the `air list` info pane no longer show `Max Retries 0` when the Jobs API leaves the task's retry limit out. For a run submitted with `air run`, the limit now comes from the run's uploaded config, shown as `3 (CLI default)` when the config doesn't set `max_retries`; it shows `N/A` when the limit can't be known (job runs, multi-container runs, legacy runs with no explicit value, or an unreadable config). ([#7623](https://github.com/databricks/cli/pull/7623)) diff --git a/acceptance/air/get-ai-runtime/output.txt b/acceptance/air/get-ai-runtime/output.txt index c84683e3f3e..57a09f7cd62 100644 --- a/acceptance/air/get-ai-runtime/output.txt +++ b/acceptance/air/get-ai-runtime/output.txt @@ -22,7 +22,7 @@ │ Status ● SUCCESS │ │ Submitted 2023-11-14 22:13 UTC │ │ Retries 0 │ -│ Max Retries 3 │ +│ Max Retries 3 (CLI default) │ │ Duration 12s │ │ Experiment my-exp │ │ MLflow Run my-run │ @@ -54,3 +54,37 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "est_percent_complete": null } } + +=== get (text, max_retries set in the config) + +>>> [CLI] air get 123 + +╭─ Configuration ────────────────────────────────────────────────╮ +│ │ +│ experiment_name: my-exp │ +│ compute: │ +│ accelerator_type: a10 │ +│ num_accelerators: 1 │ +│ max_retries: 1 │ +│ command: echo ok │ +│ │ +╰────────────────────────────────────────────────────────────────╯ + +╭─ Metadata ─────────────────────────────────────────────────────╮ +│ │ +│ Run ID 123 │ +│ Status ● SUCCESS │ +│ Submitted 2023-11-14 22:13 UTC │ +│ Retries 0 │ +│ Max Retries 1 │ +│ Duration 12s │ +│ Experiment my-exp │ +│ MLflow Run my-run │ +│ User user@example.com │ +│ Accelerators 1x A10 │ +│ Environment 4 │ +│ │ +╰────────────────────────────────────────────────────────────────╯ + +Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID] +MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 diff --git a/acceptance/air/get-ai-runtime/script b/acceptance/air/get-ai-runtime/script index 7a98ebac655..3e6e210495a 100644 --- a/acceptance/air/get-ai-runtime/script +++ b/acceptance/air/get-ai-runtime/script @@ -9,3 +9,10 @@ trace $CLI air get 123 title "get (json)" trace $CLI air get 123 -o json + +# The fake runs/get, like the real one, has no max_retries on the task, so the +# limit comes from the uploaded config: the CLI default above, the set value here. +$CLI workspace import "/Workspace/Users/user@example.com/.air/cli_launch/my-exp/my-exp_abc/training_config.yaml" --file training_config_max_retries.yaml --format AUTO --overwrite &> LOG.import-max-retries + +title "get (text, max_retries set in the config)" +trace $CLI air get 123 diff --git a/acceptance/air/get-ai-runtime/test.toml b/acceptance/air/get-ai-runtime/test.toml index 1879588b6c8..acf036047c2 100644 --- a/acceptance/air/get-ai-runtime/test.toml +++ b/acceptance/air/get-ai-runtime/test.toml @@ -12,11 +12,13 @@ Response.Body = '' # so the Environment cell instead comes from the run's environments[].spec (keyed # by the task's environment_key), which the typed SDK Run drops — `air get` reads # it with a raw request, so this route is hit twice (typed GetRun + that request). +# Like the real runs/get, the task carries no max_retries. [[Server]] Pattern = "GET /api/2.2/jobs/runs/get" Response.Body = ''' { "run_id": 123, + "run_type": "SUBMIT_RUN", "run_page_url": "https://my-workspace.cloud.databricks.test/jobs/runs/123", "creator_user_name": "user@example.com", "start_time": 1700000000000, @@ -30,7 +32,6 @@ Response.Body = ''' "task_key": "train", "run_id": 456, "attempt_number": 0, - "max_retries": 3, "environment_key": "default", "ai_runtime_task": { "experiment": "my-exp", diff --git a/acceptance/air/get-ai-runtime/training_config_max_retries.yaml b/acceptance/air/get-ai-runtime/training_config_max_retries.yaml new file mode 100644 index 00000000000..a71625352e8 --- /dev/null +++ b/acceptance/air/get-ai-runtime/training_config_max_retries.yaml @@ -0,0 +1,6 @@ +experiment_name: my-exp +compute: + accelerator_type: a10 + num_accelerators: 1 +max_retries: 1 +command: echo ok diff --git a/acceptance/air/get/output.txt b/acceptance/air/get/output.txt index 9c1fed61721..5283bbfd7f3 100644 --- a/acceptance/air/get/output.txt +++ b/acceptance/air/get/output.txt @@ -22,7 +22,7 @@ │ Status ● SUCCESS │ │ Submitted 2023-11-14 22:13 UTC │ │ Retries 0 │ -│ Max Retries 3 │ +│ Max Retries N/A │ │ Duration 12s │ │ Experiment my-exp │ │ MLflow Run my-run │ diff --git a/acceptance/air/get/test.toml b/acceptance/air/get/test.toml index 3545a34616e..dd4669e5542 100644 --- a/acceptance/air/get/test.toml +++ b/acceptance/air/get/test.toml @@ -4,12 +4,15 @@ Pattern = "HEAD /" Response.Body = '' -# A single GenAI-compute run with an experiment, GPUs, and a creator. +# A single GenAI-compute run with an experiment, GPUs, and a creator. Like the +# real runs/get, the task carries no max_retries, and its yaml_parameters leave +# max_retries unset, so the limit is unknown. [[Server]] Pattern = "GET /api/2.2/jobs/runs/get" Response.Body = ''' { "run_id": 123, + "run_type": "SUBMIT_RUN", "run_page_url": "https://my-workspace.cloud.databricks.test/jobs/runs/123", "creator_user_name": "user@example.com", "start_time": 1700000000000, @@ -20,7 +23,6 @@ Response.Body = ''' "task_key": "train", "run_id": 456, "attempt_number": 0, - "max_retries": 3, "gen_ai_compute_task": { "mlflow_experiment_name": "/Users/user@example.com/my-exp", "compute": {"gpu_type": "GPU_1xA10", "num_gpus": 1}, diff --git a/cmd/air/format.go b/cmd/air/format.go index 68c956ad4c0..2f2614dd38a 100644 --- a/cmd/air/format.go +++ b/cmd/air/format.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "math" + "slices" "strconv" "strings" "time" @@ -359,15 +360,105 @@ func gpuDisplayName(gpuType string) string { return gpuType } -// maxRetries returns the configured retry limit for the run's latest task as a -// display string: "unlimited" for the backend's -1, otherwise the count. -func maxRetries(run *jobs.Run) string { +// maxRetriesSource says where a run's retry limit was read from. +type maxRetriesSource string + +const ( + // maxRetriesReported: runs/get returned max_retries on the task. + maxRetriesReported maxRetriesSource = "reported" + // maxRetriesConfigured: the run's uploaded config sets max_retries. + maxRetriesConfigured maxRetriesSource = "configured" + // maxRetriesCLIDefault: the config leaves max_retries unset, and the CLI + // submitted its default. + maxRetriesCLIDefault maxRetriesSource = "cli_default" +) + +// runMaxRetries is a run's retry limit and where it came from. Limit -1 means +// unlimited. +type runMaxRetries struct { + Limit int + Source maxRetriesSource +} + +// resolveMaxRetries returns the retry limit of the run's latest task, or nil +// when it can't be known. configRaw is the run's training config as uploaded +// (nil when it was not found or could not be downloaded). +// +// runs/get omits max_retries on task runs, and the SDK decodes the missing field +// as 0, so a value is trusted only when the response actually carried the key. +// Otherwise the limit is read from the config, and only for one-time submit +// runs: a job run (for example a bundle deployed with `air convert-to-dabs`) +// takes its retries from the job settings, which the config doesn't track. When +// the config leaves the key unset, the CLI default is assumed only for +// ai_runtime_task runs, because every CLI that submits that task type sends 3 +// (runsubmit.go always sends max_retries). Legacy gen_ai_compute_task runs may +// predate the Python CLI's switch from 0 to 3, so their unset limit is unknown. +func resolveMaxRetries(run *jobs.Run, configRaw []byte) *runMaxRetries { + if n, ok := reportedMaxRetries(run); ok { + return &runMaxRetries{Limit: n, Source: maxRetriesReported} + } + if run.RunType != jobs.RunTypeSubmitRun { + return nil + } + n, set, ok := configMaxRetries(configRaw) + switch { + case !ok: + return nil + case set: + return &runMaxRetries{Limit: n, Source: maxRetriesConfigured} + case aiRuntimeTaskOf(run) != nil: + return &runMaxRetries{Limit: defaultMaxRetries, Source: maxRetriesCLIDefault} + default: + return nil + } +} + +// maxRetriesDisplay renders a retry limit for the text view: "N/A" when it is +// unknown, "unlimited" for -1, and a "(CLI default)" note when the config left +// it unset. +func maxRetriesDisplay(m *runMaxRetries) string { + if m == nil { + return na + } + s := strconv.Itoa(m.Limit) + if m.Limit < 0 { + s = "unlimited" + } + if m.Source == maxRetriesCLIDefault { + s += " (CLI default)" + } + return s +} + +// reportedMaxRetries returns the latest task's max_retries if the runs/get +// response carried the key. The SDK records a decoded zero value in +// ForceSendFields, which tells an explicit 0 apart from a missing field. +func reportedMaxRetries(run *jobs.Run) (int, bool) { if len(run.Tasks) == 0 { - return "0" + return 0, false + } + t := run.Tasks[len(run.Tasks)-1] + if t.MaxRetries == 0 && !slices.Contains(t.ForceSendFields, "MaxRetries") { + return 0, false + } + return t.MaxRetries, true +} + +// configMaxRetries reads max_retries from a run's config YAML: the value, whether +// the key is set, and whether the config parsed at all. It decodes only that +// key so configs written by other CLI versions still parse. +func configMaxRetries(raw []byte) (n int, set, ok bool) { + if len(raw) == 0 { + return 0, false, false + } + var cfg struct { + MaxRetries *int `yaml:"max_retries"` + } + if err := yaml.Unmarshal(raw, &cfg); err != nil { + return 0, false, false } - n := run.Tasks[len(run.Tasks)-1].MaxRetries - if n < 0 { - return "unlimited" + if cfg.MaxRetries == nil { + return 0, false, true } - return strconv.Itoa(n) + return *cfg.MaxRetries, true, true } diff --git a/cmd/air/format_test.go b/cmd/air/format_test.go index c76260d28cb..04c74c07dd1 100644 --- a/cmd/air/format_test.go +++ b/cmd/air/format_test.go @@ -1,7 +1,9 @@ package aircmd import ( + "encoding/json" "io" + "strconv" "testing" "github.com/databricks/cli/libs/cmdio" @@ -306,3 +308,93 @@ func TestTerminationReason(t *testing.T) { assert.Nil(t, terminationReason(&jobs.Run{State: &jobs.RunState{ResultState: jobs.RunResultStateSuccess, StateMessage: "done"}})) assert.Nil(t, terminationReason(&jobs.Run{State: &jobs.RunState{ResultState: jobs.RunResultStateCanceled, StateMessage: "cancelled"}})) } + +func TestMaxRetries(t *testing.T) { + submit := func(task jobs.RunTask) *jobs.Run { + return &jobs.Run{RunType: jobs.RunTypeSubmitRun, Tasks: []jobs.RunTask{task}} + } + aiRuntime := jobs.RunTask{AiRuntimeTask: &jobs.AiRuntimeTask{}} + reported := func(n int) jobs.RunTask { + return jobs.RunTask{MaxRetries: n, AiRuntimeTask: &jobs.AiRuntimeTask{}, ForceSendFields: []string{"MaxRetries"}} + } + legacy := jobs.RunTask{GenAiComputeTask: &jobs.GenAiComputeTask{}} + jobRun := &jobs.Run{RunType: jobs.RunTypeJobRun, Tasks: []jobs.RunTask{aiRuntime}} + tests := []struct { + name string + run *jobs.Run + config string + want string + }{ + {"submit, config sets 1", submit(aiRuntime), "max_retries: 1\n", "1"}, + {"submit, config sets 0", submit(aiRuntime), "max_retries: 0\n", "0"}, + {"submit, config sets unlimited", submit(aiRuntime), "max_retries: -1\n", "unlimited"}, + {"submit, config unset uses the CLI default", submit(aiRuntime), "experiment_name: e\n", "3 (CLI default)"}, + {"submit, no config", submit(aiRuntime), "", "N/A"}, + {"submit, unparsable config", submit(aiRuntime), "[not: a map", "N/A"}, + {"legacy submit, config unset is unknown", submit(legacy), "experiment_name: e\n", "N/A"}, + {"legacy submit, config sets 2", submit(legacy), "max_retries: 2\n", "2"}, + {"job run ignores the config", jobRun, "max_retries: 1\n", "N/A"}, + {"job run, config unset", jobRun, "experiment_name: e\n", "N/A"}, + {"run type unknown", &jobs.Run{Tasks: []jobs.RunTask{aiRuntime}}, "max_retries: 1\n", "N/A"}, + {"server reports 0 over the config", submit(reported(0)), "max_retries: 2\n", "0"}, + {"server reports 5 on a job run", &jobs.Run{RunType: jobs.RunTypeJobRun, Tasks: []jobs.RunTask{reported(5)}}, "", "5"}, + {"server reports unlimited", submit(reported(-1)), "", "unlimited"}, + {"no tasks", &jobs.Run{RunType: jobs.RunTypeSubmitRun}, "", "N/A"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var raw []byte + if tt.config != "" { + raw = []byte(tt.config) + } + assert.Equal(t, tt.want, maxRetriesDisplay(resolveMaxRetries(tt.run, raw))) + }) + } +} + +func TestResolveMaxRetriesSource(t *testing.T) { + run := &jobs.Run{RunType: jobs.RunTypeSubmitRun, Tasks: []jobs.RunTask{{AiRuntimeTask: &jobs.AiRuntimeTask{}}}} + assert.Equal(t, &runMaxRetries{Limit: 1, Source: maxRetriesConfigured}, resolveMaxRetries(run, []byte("max_retries: 1\n"))) + assert.Equal(t, &runMaxRetries{Limit: 3, Source: maxRetriesCLIDefault}, resolveMaxRetries(run, []byte("command: x\n"))) + run.Tasks[0].MaxRetries = 2 + assert.Equal(t, &runMaxRetries{Limit: 2, Source: maxRetriesReported}, resolveMaxRetries(run, nil)) +} + +// The runs/get bodies below are trimmed copies of real workspace responses: +// task runs carry no max_retries key at all. +func TestMaxRetriesFromDecodedRunGet(t *testing.T) { + decode := func(body string) *jobs.Run { + var run jobs.Run + require.NoError(t, json.Unmarshal([]byte(body), &run)) + return &run + } + display := func(run *jobs.Run, config []byte) string { + return maxRetriesDisplay(resolveMaxRetries(run, config)) + } + // Run 111111111111111: submitted with max_retries 1, failed, retried once. + aiRuntime := decode(`{"run_id": 111111111111111, "run_type": "SUBMIT_RUN", "tasks": [ + {"task_key": "t", "run_id": 222222222222222, "attempt_number": 0, "ai_runtime_task": {"experiment": "e"}}, + {"task_key": "t", "run_id": 333333333333333, "attempt_number": 1, "ai_runtime_task": {"experiment": "e"}}]}`) + assert.Equal(t, "1", display(aiRuntime, []byte("experiment_name: e\nmax_retries: 1\n"))) + assert.Equal(t, "3 (CLI default)", display(aiRuntime, []byte("experiment_name: e\n"))) + + // A multi-container run has no deployment-level command_path, so the config + // path is unknown and nothing is downloaded. + multi := decode(`{"run_id": 1, "run_type": "SUBMIT_RUN", "tasks": [{"task_key": "t", "run_id": 2, + "ai_runtime_task": {"experiment": "e", "deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}}]}}]}`) + var data getData + enrichFromAiRuntimeTask(multi, &data) + assert.Empty(t, data.TrainingConfigPath) + assert.Equal(t, "N/A", display(multi, nil)) + + // A legacy run as the Python CLI submitted it: yaml_parameters is the user's + // raw YAML, and max_retries is sent only when > 0. + legacyYAML := "experiment_name: my-exp\ncompute:\n accelerator_type: a10\n num_accelerators: 1\ncommand: python train.py\n" + legacy := decode(`{"run_id": 1, "run_type": "SUBMIT_RUN", "tasks": [{"task_key": "train", "run_id": 2, "attempt_number": 0, + "gen_ai_compute_task": {"mlflow_experiment_name": "/Users/u/my-exp", "yaml_parameters": ` + strconv.Quote(legacyYAML) + `}}]}`) + _, raw := configYAML(t.Context(), nil, genAIComputeTask(legacy)) + assert.Equal(t, "N/A", display(legacy, raw)) + + explicitZero := decode(`{"run_type": "SUBMIT_RUN", "tasks": [{"max_retries": 0, "ai_runtime_task": {}}]}`) + assert.Equal(t, "0", display(explicitZero, []byte("experiment_name: e\n"))) +} diff --git a/cmd/air/get.go b/cmd/air/get.go index bac778ded51..963ec873f69 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -52,7 +52,6 @@ type getData struct { UserDisplay string `json:"-"` AcceleratorsDisplay string `json:"-"` EnvironmentDisplay string `json:"-"` - MaxRetriesDisplay string `json:"-"` // ProgressDisplay is the pre-rendered "Progress" cell ("45% · ~2h 15m left"), // set only for a running run with an estimable remaining time. ProgressDisplay string `json:"-"` @@ -292,6 +291,5 @@ func buildGetData(run *jobs.Run) getData { // version needs a raw GetRun read (the typed SDK Run drops it), so it is not // filled here alongside the fields read straight off the run. data.EnvironmentDisplay = na - data.MaxRetriesDisplay = maxRetries(run) return data } diff --git a/cmd/air/list_detail.go b/cmd/air/list_detail.go index 59adaf93316..887fca8dd7d 100644 --- a/cmd/air/list_detail.go +++ b/cmd/air/list_detail.go @@ -24,6 +24,9 @@ func runDetailText(ctx context.Context, w *databricks.WorkspaceClient, runID int data := buildGetData(run) data.DashboardURL = dashboardURL(w.Config.Host, runID, workspaceIDForRunLink(ctx, w)) + if findForEachTask(run) == nil && genAIComputeTask(run) == nil { + enrichFromAiRuntimeTask(run, &data) + } ids := mlflowIDs(ctx, w, run) if ids != nil { url := mlflowLogsURL(w.Config.Host, ids) diff --git a/cmd/air/list_detail_test.go b/cmd/air/list_detail_test.go index 88939b9f363..5f880bc1481 100644 --- a/cmd/air/list_detail_test.go +++ b/cmd/air/list_detail_test.go @@ -145,3 +145,54 @@ func TestUnnamedOlderRetryLabelFitsTheListColumnAndIsWholeInTheInfoPane(t *testi assert.Equal(t, mlflowColWidth, lipgloss.Width(rows[0].MLflowLabel)) assert.Contains(t, info, mlflowRunID+" (retry 0)") } + +func TestRunDetailTextReadsTheUploadedConfig(t *testing.T) { + const configPath = "/Workspace/Users/me@example.com/.air/cli_launch/exp/exp_abc/training_config.yaml" + tests := []struct { + name string + runType string + config string // "" means the download fails + want string + }{ + {"configured limit", "SUBMIT_RUN", "experiment_name: exp\nmax_retries: 1\n", "Max Retries 1 "}, + {"unset limit", "SUBMIT_RUN", "experiment_name: exp\n", "Max Retries 3 (CLI default)"}, + {"config download fails", "SUBMIT_RUN", "", "Max Retries N/A"}, + {"job run", "JOB_RUN", "experiment_name: exp\nmax_retries: 1\n", "Max Retries N/A"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Given an ai_runtime run whose runs/get omits max_retries, as the + // Jobs API does. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/2.2/jobs/runs/get": + _, _ = w.Write([]byte(`{"run_id": 123, "run_type": "` + tt.runType + `", + "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED"}, + "tasks": [{"run_id": 456, "attempt_number": 0, "ai_runtime_task": {"experiment": "/Users/me@example.com/exp", + "deployments": [{"command_path": "/Workspace/Users/me@example.com/.air/cli_launch/exp/exp_abc/command.sh", + "compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}}]}}]}`)) + case "/api/2.0/workspace/export": + if tt.config != "" && r.URL.Query().Get("path") == configPath { + _, _ = w.Write([]byte(tt.config)) + return + } + w.WriteHeader(http.StatusNotFound) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + + // When the list's info pane renders it. + got, err := runDetailText(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL), 123) + require.NoError(t, err) + + // Then it shows the same config box and limit as `air get`. + assert.Contains(t, got, tt.want) + assert.Contains(t, got, "1x A10") + if tt.config != "" { + assert.Contains(t, got, "experiment_name: exp") + } + }) + } +} diff --git a/cmd/air/render.go b/cmd/air/render.go index 755e8540fb9..acdaf7f7c78 100644 --- a/cmd/air/render.go +++ b/cmd/air/render.go @@ -104,13 +104,15 @@ func renderRunText(ctx context.Context, out io.Writer, w *databricks.WorkspaceCl // with no serverless environment) stays "N/A". data.EnvironmentDisplay = orNA(aiRuntimeEnvironmentVersion(ctx, w, run.RunId)) + configBody, configRaw := resolveConfigYAML(ctx, w, run, data) + view := runView{ runID: data.RunID, dashboardURL: data.DashboardURL, status: data.Status, submitted: data.SubmittedDisplay, retries: data.AttemptNumber, - maxRetries: data.MaxRetriesDisplay, + maxRetries: maxRetriesDisplay(resolveMaxRetries(run, configRaw)), duration: data.DurationDisplay, progress: data.ProgressDisplay, experiment: data.ExperimentDisplay, @@ -130,7 +132,7 @@ func renderRunText(ctx context.Context, out io.Writer, w *databricks.WorkspaceCl } var sections []string - if body := colorizeConfig(p, resolveConfigYAML(ctx, w, run, data)); body != "" { + if body := colorizeConfig(p, configBody); body != "" { sections = append(sections, renderBox(p, configBoxTitle, body)) } sections = append(sections, renderBox(p, metadataBoxTitle, renderFields(p, colorOn, view))) @@ -199,33 +201,35 @@ func aiRuntimeEnvironmentVersion(ctx context.Context, w *databricks.WorkspaceCli } // resolveConfigYAML returns the config box body: from the downloaded config file -// when we have its path, else from the legacy task. -func resolveConfigYAML(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run, data *getData) string { +// when we have its path, else from the legacy task. It also returns the raw +// config the body came from, or nil when the body is synthesized or empty. +func resolveConfigYAML(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run, data *getData) (string, []byte) { if data.TrainingConfigPath != "" { if raw := downloadConfig(ctx, w, data.TrainingConfigPath); len(raw) > 0 { - return strings.TrimRight(reformatYAMLForDisplay(raw), "\n") + return strings.TrimRight(reformatYAMLForDisplay(raw), "\n"), raw } } if task := genAIComputeTask(run); task != nil { return configYAML(ctx, w, task) } - return "" + return "", nil } // configYAML returns the run's resolved training config as YAML for the box. The // full config (including the command/script) lives in the run's parameters, not // the structured task fields, so we prefer the inline parameters, then the // parameters file, and only synthesize a minimal config as a last resort. -func configYAML(ctx context.Context, w *databricks.WorkspaceClient, task *jobs.GenAiComputeTask) string { +func configYAML(ctx context.Context, w *databricks.WorkspaceClient, task *jobs.GenAiComputeTask) (string, []byte) { if task.YamlParameters != "" { - return strings.TrimRight(reformatYAMLForDisplay([]byte(task.YamlParameters)), "\n") + raw := []byte(task.YamlParameters) + return strings.TrimRight(reformatYAMLForDisplay(raw), "\n"), raw } if task.YamlParametersFilePath != "" { if raw := downloadConfig(ctx, w, task.YamlParametersFilePath); len(raw) > 0 { - return strings.TrimRight(reformatYAMLForDisplay(raw), "\n") + return strings.TrimRight(reformatYAMLForDisplay(raw), "\n"), raw } } - return synthConfigYAML(task) + return synthConfigYAML(task), nil } // downloadConfig fetches the run's training-config file, returning nil on diff --git a/cmd/air/render_test.go b/cmd/air/render_test.go index 66e5b09deb7..cd25947529a 100644 --- a/cmd/air/render_test.go +++ b/cmd/air/render_test.go @@ -28,7 +28,7 @@ func TestConfigYAML(t *testing.T) { task := &jobs.GenAiComputeTask{ YamlParameters: "experiment_name: my-exp\ncompute:\n accelerator_type: a10\n num_accelerators: 1\ncommand: \"for i in $(seq 1 3); do echo $i; done\\n\"\n", } - got := configYAML(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient, task) + got, _ := configYAML(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient, task) assert.Contains(t, got, "experiment_name: my-exp") assert.Contains(t, got, "accelerator_type: a10") assert.Contains(t, got, "command: |") @@ -41,7 +41,9 @@ func TestConfigYAML(t *testing.T) { Download(mock.Anything, "/Workspace/cfg.yaml"). Return(io.NopCloser(strings.NewReader("experiment_name: from-file\n")), nil) task := &jobs.GenAiComputeTask{YamlParametersFilePath: "/Workspace/cfg.yaml"} - assert.Equal(t, "experiment_name: from-file", configYAML(ctx, m.WorkspaceClient, task)) + got, raw := configYAML(ctx, m.WorkspaceClient, task) + assert.Equal(t, "experiment_name: from-file", got) + assert.Equal(t, "experiment_name: from-file\n", string(raw)) }) t.Run("falls back to a synthesized config", func(t *testing.T) { @@ -49,7 +51,7 @@ func TestConfigYAML(t *testing.T) { MlflowExperimentName: "/Users/me@example.com/exp", Compute: &jobs.ComputeConfig{GpuType: "a10", NumGpus: 1}, } - got := configYAML(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient, task) + got, _ := configYAML(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient, task) assert.Contains(t, got, "experiment_name: exp") assert.NotContains(t, got, "command") }) @@ -65,20 +67,24 @@ func TestResolveConfigYAML(t *testing.T) { Return(io.NopCloser(strings.NewReader("experiment_name: from-air\n")), nil) run := &jobs.Run{Tasks: []jobs.RunTask{{}}} data := &getData{TrainingConfigPath: "/Workspace/run/training_config.yaml"} - assert.Equal(t, "experiment_name: from-air", resolveConfigYAML(ctx, m.WorkspaceClient, run, data)) + got, raw := resolveConfigYAML(ctx, m.WorkspaceClient, run, data) + assert.Equal(t, "experiment_name: from-air", got) + assert.Equal(t, "experiment_name: from-air\n", string(raw)) }) t.Run("falls back to the legacy task when no path is set", func(t *testing.T) { run := &jobs.Run{Tasks: []jobs.RunTask{{GenAiComputeTask: &jobs.GenAiComputeTask{ MlflowExperimentName: "/Users/me@example.com/exp", }}}} - got := resolveConfigYAML(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient, run, &getData{}) + got, _ := resolveConfigYAML(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient, run, &getData{}) assert.Contains(t, got, "experiment_name: exp") }) t.Run("empty when neither source is present", func(t *testing.T) { run := &jobs.Run{Tasks: []jobs.RunTask{{}}} - assert.Empty(t, resolveConfigYAML(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient, run, &getData{})) + got, raw := resolveConfigYAML(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient, run, &getData{}) + assert.Empty(t, got) + assert.Nil(t, raw) }) }