diff --git a/.nextchanges/air/get-json-details.md b/.nextchanges/air/get-json-details.md new file mode 100644 index 00000000000..ccb20f01ecf --- /dev/null +++ b/.nextchanges/air/get-json-details.md @@ -0,0 +1 @@ +* `air get -o json` includes run details including flat `accelerator_type` and `num_accelerators`. Use `--include-config` to include stored config, which may contain plaintext `env_variables` and parameters; downloads over 1 MiB are omitted, and config-dependent `max_retries` is otherwise null. ([#7624](https://github.com/databricks/cli/pull/7624)) diff --git a/acceptance/air/get-ai-runtime/output.txt b/acceptance/air/get-ai-runtime/output.txt index 57a09f7cd62..b63a4f08d40 100644 --- a/acceptance/air/get-ai-runtime/output.txt +++ b/acceptance/air/get-ai-runtime/output.txt @@ -51,7 +51,48 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs", "est_remaining_seconds": null, - "est_percent_complete": null + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": "4", + "base_environment": null, + "max_retries": null, + "mlflow_run_name": "my-run", + "sweep": null + } +} + +=== get (json, include config) + +>>> [CLI] air get 123 -o json --include-config +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "run_id": "123", + "status": "SUCCESS", + "started_at": "[TIMESTAMP]", + "duration_seconds": 12, + "attempt_number": 0, + "experiment_name": "my-exp", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", + "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs", + "est_remaining_seconds": null, + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": "4", + "base_environment": null, + "max_retries": { + "limit": 3, + "source": "cli_default" + }, + "mlflow_run_name": "my-run", + "sweep": null, + "config": "experiment_name: my-exp\ncompute:\n accelerator_type: a10\n num_accelerators: 1\ncommand: |-\n for i in $(seq 1 10); do\n echo \"step $i\"\n done\n", + "config_source": "uploaded" } } @@ -88,3 +129,64 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID] MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 + +=== get (json, max_retries set in the config) + +>>> [CLI] air get 123 -o json +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "run_id": "123", + "status": "SUCCESS", + "started_at": "[TIMESTAMP]", + "duration_seconds": 12, + "attempt_number": 0, + "experiment_name": "my-exp", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", + "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs", + "est_remaining_seconds": null, + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": "4", + "base_environment": null, + "max_retries": null, + "mlflow_run_name": "my-run", + "sweep": null + } +} + +=== get (json, include config, max_retries set) + +>>> [CLI] air get 123 -o json --include-config +{ + "v": 1, + "ts": "[TIMESTAMP]", + "data": { + "run_id": "123", + "status": "SUCCESS", + "started_at": "[TIMESTAMP]", + "duration_seconds": 12, + "attempt_number": 0, + "experiment_name": "my-exp", + "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", + "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs", + "est_remaining_seconds": null, + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": "4", + "base_environment": null, + "max_retries": { + "limit": 1, + "source": "configured" + }, + "mlflow_run_name": "my-run", + "sweep": null, + "config": "experiment_name: my-exp\ncompute:\n accelerator_type: a10\n num_accelerators: 1\nmax_retries: 1\ncommand: echo ok\n", + "config_source": "uploaded" + } +} diff --git a/acceptance/air/get-ai-runtime/script b/acceptance/air/get-ai-runtime/script index 3e6e210495a..a73d6742a95 100644 --- a/acceptance/air/get-ai-runtime/script +++ b/acceptance/air/get-ai-runtime/script @@ -10,9 +10,18 @@ trace $CLI air get 123 title "get (json)" trace $CLI air get 123 -o json +title "get (json, include config)" +trace $CLI air get 123 -o json --include-config + # The fake runs/get, like the real one, has no max_retries on the task, so the # limit comes from the uploaded config: the CLI default above, the set value here. $CLI workspace import "/Workspace/Users/user@example.com/.air/cli_launch/my-exp/my-exp_abc/training_config.yaml" --file training_config_max_retries.yaml --format AUTO --overwrite &> LOG.import-max-retries title "get (text, max_retries set in the config)" trace $CLI air get 123 + +title "get (json, max_retries set in the config)" +trace $CLI air get 123 -o json + +title "get (json, include config, max_retries set)" +trace $CLI air get 123 -o json --include-config diff --git a/acceptance/air/get-pending/output.txt b/acceptance/air/get-pending/output.txt index 3a5f9a72eda..c838d560527 100644 --- a/acceptance/air/get-pending/output.txt +++ b/acceptance/air/get-pending/output.txt @@ -14,6 +14,17 @@ "mlflow_url": null, "status_message": "Waiting for accelerator compute capacity to become available...", "est_remaining_seconds": null, - "est_percent_complete": null + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": null, + "base_environment": null, + "max_retries": { + "limit": 0, + "source": "reported" + }, + "mlflow_run_name": null, + "sweep": null } } diff --git a/acceptance/air/get/output.txt b/acceptance/air/get/output.txt index 5283bbfd7f3..e82b0022e9f 100644 --- a/acceptance/air/get/output.txt +++ b/acceptance/air/get/output.txt @@ -51,7 +51,15 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 "dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]", "mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs", "est_remaining_seconds": null, - "est_percent_complete": null + "est_percent_complete": null, + "user": "user@example.com", + "accelerator_type": "GPU_1xA10", + "num_accelerators": 1, + "environment_version": null, + "base_environment": null, + "max_retries": null, + "mlflow_run_name": "my-run", + "sweep": null } } diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index 7b37a4c83ac..5091bb160a3 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -96,7 +96,8 @@ Usage: databricks air get JOB_RUN_ID [flags] Flags: - -h, --help help for get + -h, --help help for get + --include-config Include stored config in JSON (downloads over 1 MiB are omitted; may contain plaintext env_variables and parameters) Global Flags: --debug enable debug logging diff --git a/cmd/air/format.go b/cmd/air/format.go index 2f2614dd38a..d73c2d8a620 100644 --- a/cmd/air/format.go +++ b/cmd/air/format.go @@ -326,19 +326,6 @@ func stripExperimentUserPrefix(name string) string { return name } -// accelerators returns a short description of the GPUs the run uses, such as -// "8x H100", or an empty string if the run has no GPU compute attached. -func accelerators(run *jobs.Run) string { - if len(run.Tasks) == 0 { - return "" - } - task := run.Tasks[0].GenAiComputeTask - if task == nil || task.Compute == nil { - return "" - } - return acceleratorLabel(task.Compute.GpuType, task.Compute.NumGpus) -} - // acceleratorLabel renders a GPU count and type as "8x H100", "" for none, or // the count alone ("8x") when the type is unrecognized. func acceleratorLabel(gpuType string, count int) string { @@ -376,8 +363,8 @@ const ( // runMaxRetries is a run's retry limit and where it came from. Limit -1 means // unlimited. type runMaxRetries struct { - Limit int - Source maxRetriesSource + Limit int `json:"limit"` + Source maxRetriesSource `json:"source"` } // resolveMaxRetries returns the retry limit of the run's latest task, or nil diff --git a/cmd/air/format_test.go b/cmd/air/format_test.go index 04c74c07dd1..306e0bc3e1c 100644 --- a/cmd/air/format_test.go +++ b/cmd/air/format_test.go @@ -213,20 +213,6 @@ func TestReformatYAMLForDisplay(t *testing.T) { assert.Equal(t, "\tnot: valid: yaml:", reformatYAMLForDisplay([]byte("\tnot: valid: yaml:"))) } -func TestAccelerators(t *testing.T) { - assert.Empty(t, accelerators(&jobs.Run{})) - assert.Empty(t, accelerators(&jobs.Run{Tasks: []jobs.RunTask{{}}})) - assert.Empty(t, accelerators(&jobs.Run{Tasks: []jobs.RunTask{{ - GenAiComputeTask: &jobs.GenAiComputeTask{}, - }}})) - assert.Empty(t, accelerators(&jobs.Run{Tasks: []jobs.RunTask{{ - GenAiComputeTask: &jobs.GenAiComputeTask{Compute: &jobs.ComputeConfig{NumGpus: 0}}, - }}})) - assert.Equal(t, "8x H100", accelerators(&jobs.Run{Tasks: []jobs.RunTask{{ - GenAiComputeTask: &jobs.GenAiComputeTask{Compute: &jobs.ComputeConfig{NumGpus: 8, GpuType: "GPU_8xH100"}}, - }}})) -} - func TestAcceleratorLabel(t *testing.T) { assert.Empty(t, acceleratorLabel("GPU_8xH100", 0)) assert.Equal(t, "8x H100", acceleratorLabel("GPU_8xH100", 8)) diff --git a/cmd/air/get.go b/cmd/air/get.go index 963ec873f69..76664aaf24d 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -6,11 +6,14 @@ import ( "fmt" "path" "strconv" + "strings" + "sync" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/config" @@ -41,11 +44,40 @@ type getData struct { ESTRemainingSeconds *int64 `json:"est_remaining_seconds"` ESTPercentComplete *int `json:"est_percent_complete"` - // The fields below are pre-rendered text-view cells, excluded from JSON - // (matching `air get --json`). Each shows "N/A" when its value is - // missing. The styled single-run renderer (render.go) consumes them; the - // Run ID, Status, and MLflow Run cells it draws are styled and hyperlinked - // there rather than stored here. + // The fields below carry the values of the text view's Configuration and + // Metadata boxes and its sweep summary. Missing values are null; config-based + // retry limits require opting in to reading the stored config. + User *string `json:"user"` + AcceleratorType *string `json:"accelerator_type"` + NumAccelerators *int `json:"num_accelerators"` + // EnvironmentVersion and BaseEnvironment come from the run's serverless + // environment spec; at most one is set. + EnvironmentVersion *string `json:"environment_version"` + BaseEnvironment *string `json:"base_environment"` + // MaxRetries is the run's retry limit and its source (see resolveMaxRetries). + MaxRetries *runMaxRetries `json:"max_retries"` + // Config is the stored run config the Configuration box shows, before display + // reformatting. ConfigSource names where it came from: "uploaded" (the + // training_config.yaml `air run` uploads after applying overrides and + // stripping comments), or "yaml_parameters" / "yaml_parameters_file" (legacy + // runs, a different schema). A config synthesized for display is not exposed. + Config *string `json:"-"` + ConfigSource *string `json:"-"` + // MLflowRunName is the MLflow run's name, which the MLflow Run cell shows. + MLflowRunName *string `json:"mlflow_run_name"` + // Sweep is set for foreach runs; its table replaces the single-run view. + Sweep *sweepInfo `json:"sweep"` + + // configBody is the Configuration box body, which may be synthesized for a + // legacy run without stored parameters. + configBody string + // detailsResolved records that enrichRunDetails has run. + detailsResolved bool + + // The fields below are pre-rendered text-view cells, excluded from JSON. + // Each shows "N/A" when its value is missing. The styled single-run renderer + // (render.go) consumes them; the Run ID, Status, and MLflow Run cells it draws + // are styled and hyperlinked there rather than stored here. SubmittedDisplay string `json:"-"` DurationDisplay string `json:"-"` ExperimentDisplay string `json:"-"` @@ -58,8 +90,6 @@ type getData struct { DisplayStatus string `json:"-"` // TrainingConfigPath is the run's config file, downloaded for the config box. TrainingConfigPath string `json:"-"` - // Sweep replaces the single-run view for foreach runs. - Sweep *sweepInfo `json:"-"` } // getTemplate is the text-mode layout for a sweep (foreach) run. Single runs are @@ -118,6 +148,7 @@ func credentialError(ctx context.Context, w *databricks.WorkspaceClient, err err // newGetCommand returns the `air get JOB_RUN_ID` command, which shows status, // configuration, and timing details for a specific run. func newGetCommand() *cobra.Command { + var includeConfig bool cmd := &cobra.Command{ Use: "get JOB_RUN_ID", Args: root.ExactArgs(1), @@ -130,6 +161,8 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, }, } + cmd.Flags().BoolVar(&includeConfig, "include-config", false, "Include stored config in JSON (downloads over 1 MiB are omitted; may contain plaintext env_variables and parameters)") + // Resolve and authenticate the workspace client up front so an auth failure // fails fast here, before any run status or config is fetched or printed. // ErrAlreadyPrinted passes through (it was handled upstream); other failures @@ -144,6 +177,9 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() + if includeConfig && root.OutputType(cmd) == flags.OutputText { + return renderError(ctx, cmd, "INVALID_ARGS", "PERMANENT", false, errors.New("--include-config requires -o json")) + } w := cmdctx.WorkspaceClient(ctx) runID, err := strconv.ParseInt(args[0], 10, 64) @@ -206,11 +242,27 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, } if task := findForEachTask(run); task != nil { data.Sweep = buildSweepInfo(ctx, w, task) - } else if genAIComputeTask(run) == nil { - enrichFromAiRuntimeTask(run, &data) + } else { + if genAIComputeTask(run) == nil { + enrichFromAiRuntimeTask(run, &data) + } + enrichCtx := ctx + if root.OutputType(cmd) != flags.OutputText { + // A failed read already shows as a null field; keep stderr quiet + // for callers that merge it into stdout. + enrichCtx = withQuietEnrichment(ctx) + } + enrichRunDetails(enrichCtx, w, run, &data, ids, root.OutputType(cmd) == flags.OutputText || includeConfig) } if root.OutputType(cmd) != flags.OutputText { + if includeConfig { + return renderEnvelope(ctx, struct { + getData + Config *string `json:"config"` + ConfigSource *string `json:"config_source"` + }{data, data.Config, data.ConfigSource}) + } return renderEnvelope(ctx, data) } @@ -243,9 +295,7 @@ func enrichFromAiRuntimeTask(run *jobs.Run, data *getData) { if d.CommandPath != "" { data.TrainingConfigPath = path.Join(path.Dir(d.CommandPath), trainingConfigName) } - if a := acceleratorLabel(string(d.Compute.AcceleratorType), d.Compute.AcceleratorCount); a != "" { - data.AcceleratorsDisplay = a - } + setAccelerators(data, string(d.Compute.AcceleratorType), d.Compute.AcceleratorCount) } if task.Experiment != "" { exp := stripExperimentUserPrefix(task.Experiment) @@ -285,11 +335,92 @@ func buildGetData(run *jobs.Run) getData { if data.ExperimentName != nil { data.ExperimentDisplay = *data.ExperimentName } + data.User = nilIfEmpty(run.CreatorUserName) data.UserDisplay = orNA(run.CreatorUserName) - data.AcceleratorsDisplay = orNA(accelerators(run)) + data.AcceleratorsDisplay = na + accelType, count := jobCompute(run) + setAccelerators(&data, accelType, count) // EnvironmentDisplay is resolved at render time: the serverless environment // version needs a raw GetRun read (the typed SDK Run drops it), so it is not // filled here alongside the fields read straight off the run. data.EnvironmentDisplay = na return data } + +// setAccelerators records GPU compute in the same nullable shape as air list. +func setAccelerators(data *getData, accelType string, count int) { + data.AcceleratorType = nilIfEmpty(accelType) + if count != 0 { + data.NumAccelerators = new(count) + } else { + data.NumAccelerators = nil + } + data.AcceleratorsDisplay = orNA(acceleratorLabel(accelType, count)) +} + +// enrichRunDetails resolves the single-run values that need requests beyond the +// typed GetRun: the environment (raw GetRun), the stored config (download) and +// the retry limit and the MLflow run name. When readConfig is false, only a +// server-reported retry limit is available. Each read is best-effort. +func enrichRunDetails(ctx context.Context, w *databricks.WorkspaceClient, run *jobs.Run, data *getData, ids *mlflowIdentifiers, readConfig bool) { + data.detailsResolved = true + + // The reads are independent, so run them concurrently. + var ( + wg sync.WaitGroup + version, base string + body string + raw []byte + mlflowName string + ) + wg.Go(func() { version, base = aiRuntimeEnvironment(ctx, w, run.RunId) }) + if readConfig { + wg.Go(func() { body, raw = resolveConfigYAML(ctx, w, run, data) }) + } + if ids != nil { + wg.Go(func() { mlflowName = fetchMLflowRunName(ctx, w, ids.RunID) }) + } + wg.Wait() + + data.EnvironmentVersion = nilIfEmpty(version) + data.BaseEnvironment = nilIfEmpty(base) + data.EnvironmentDisplay = orNA(version) + data.configBody = body + if len(raw) > 0 { + // Normalize CRLF so the machine-readable output is identical for the + // same config uploaded from Windows and Unix checkouts. + cfg := strings.ReplaceAll(string(raw), "\r\n", "\n") + data.Config = &cfg + data.ConfigSource = configSource(run, data) + } + data.MaxRetries = resolveMaxRetries(run, raw) + data.MLflowRunName = nilIfEmpty(mlflowName) +} + +// configSource names where resolveConfigYAML read a stored config from. +func configSource(run *jobs.Run, data *getData) *string { + source := "uploaded" + if task := genAIComputeTask(run); task != nil && data.TrainingConfigPath == "" { + source = "yaml_parameters_file" + if task.YamlParameters != "" { + source = "yaml_parameters" + } + } + return &source +} + +type quietEnrichmentKey struct{} + +// withQuietEnrichment demotes best-effort enrichment failures from warnings to +// debug logs. +func withQuietEnrichment(ctx context.Context) context.Context { + return context.WithValue(ctx, quietEnrichmentKey{}, true) +} + +func enrichmentWarnf(ctx context.Context, format string, v ...any) { + if quiet, _ := ctx.Value(quietEnrichmentKey{}).(bool); quiet { + log.Debugf(ctx, format, v...) + return + } + log.Warnf(ctx, format, v...) +} diff --git a/cmd/air/get_test.go b/cmd/air/get_test.go index c6e984c3b44..c70387f6d76 100644 --- a/cmd/air/get_test.go +++ b/cmd/air/get_test.go @@ -324,7 +324,9 @@ func TestBuildGetDataTerminationReasonJSON(t *testing.T) { assert.JSONEq(t, `{ "run_id":"5","status":"FAILED","started_at":null,"duration_seconds":null, "attempt_number":0,"experiment_name":null,"dashboard_url":"","mlflow_url":null, - "termination_reason":"out of memory","est_remaining_seconds":null,"est_percent_complete":null + "termination_reason":"out of memory","est_remaining_seconds":null,"est_percent_complete":null, + "user":null,"accelerator_type":null,"num_accelerators":null,"environment_version":null,"base_environment":null, + "max_retries":null,"mlflow_run_name":null,"sweep":null }`, string(body)) canceled := buildGetData(&jobs.Run{RunId: 6, State: &jobs.RunState{ResultState: jobs.RunResultStateCanceled, StateMessage: "user canceled"}}) @@ -346,7 +348,9 @@ func TestBuildGetDataWaitingForComputeStatus(t *testing.T) { assert.JSONEq(t, `{ "run_id":"8","status":"PENDING","started_at":null,"duration_seconds":null, "attempt_number":0,"experiment_name":null,"dashboard_url":"","mlflow_url":null, - "est_remaining_seconds":null,"est_percent_complete":null + "est_remaining_seconds":null,"est_percent_complete":null, + "user":null,"accelerator_type":null,"num_accelerators":null,"environment_version":null,"base_environment":null, + "max_retries":null,"mlflow_run_name":null,"sweep":null }`, string(body)) } @@ -400,3 +404,223 @@ func TestBuildGetDataEmpty(t *testing.T) { assert.Equal(t, na, d.UserDisplay) assert.Equal(t, na, d.AcceleratorsDisplay) } + +func TestBuildGetDataMetadata(t *testing.T) { + t.Run("legacy task", func(t *testing.T) { + d := buildGetData(&jobs.Run{ + CreatorUserName: "me@example.com", + Tasks: []jobs.RunTask{{GenAiComputeTask: &jobs.GenAiComputeTask{ + Compute: &jobs.ComputeConfig{GpuType: "GPU_8xH100", NumGpus: 8}, + }}}, + }) + assert.Equal(t, "me@example.com", *d.User) + assert.Equal(t, new("GPU_8xH100"), d.AcceleratorType) + assert.Equal(t, new(8), d.NumAccelerators) + assert.Equal(t, "8x H100", d.AcceleratorsDisplay) + }) + + t.Run("ai_runtime task", func(t *testing.T) { + d := buildGetData(&jobs.Run{Tasks: []jobs.RunTask{{AiRuntimeTask: &jobs.AiRuntimeTask{ + Deployments: []jobs.DeploymentSpec{{ + Compute: jobs.ComputeSpec{AcceleratorType: jobs.ComputeSpecAcceleratorTypeGpu1xA10, AcceleratorCount: 1}, + }}, + }}}}) + assert.Equal(t, new("GPU_1xA10"), d.AcceleratorType) + assert.Equal(t, new(1), d.NumAccelerators) + assert.Equal(t, "1x A10", d.AcceleratorsDisplay) + }) + + t.Run("no compute or creator stays null", func(t *testing.T) { + d := buildGetData(&jobs.Run{Tasks: []jobs.RunTask{{GenAiComputeTask: &jobs.GenAiComputeTask{ + Compute: &jobs.ComputeConfig{NumGpus: 0}, + }}}}) + assert.Nil(t, d.User) + assert.Nil(t, d.AcceleratorType) + assert.Nil(t, d.NumAccelerators) + assert.Equal(t, na, d.AcceleratorsDisplay) + assert.Equal(t, na, d.UserDisplay) + }) +} + +func TestGetAcceleratorJSONMatchesListWhenCountMissing(t *testing.T) { + run := &jobs.Run{Tasks: []jobs.RunTask{{GenAiComputeTask: &jobs.GenAiComputeTask{Compute: &jobs.ComputeConfig{GpuType: "GPU_1xA10"}}}}} + data := buildGetData(run) + encoded, err := json.Marshal(data) + require.NoError(t, err) + var fields map[string]any + require.NoError(t, json.Unmarshal(encoded, &fields)) + assert.Equal(t, "GPU_1xA10", fields["accelerator_type"]) + assert.Nil(t, fields["num_accelerators"]) +} + +func TestEnrichRunDetails(t *testing.T) { + const uploaded = "/Workspace/run/training_config.yaml" + submitAiRuntime := `"run_type": "SUBMIT_RUN", "tasks": [{"ai_runtime_task": {}}]` + tests := []struct { + name string + run string + config string // served at uploaded; empty means not found + mlflowName string // empty means the MLflow lookup fails + wantVersion *string + wantBase *string + wantEnvCell string + wantSource *string + wantRetries *runMaxRetries + wantBody string + }{ + { + name: "uploaded config and environment version", + run: `{"run_id": 1, "environments": [{"environment_key": "default", "spec": {"environment_version": "6"}}], ` + submitAiRuntime + `}`, + config: "max_retries: 1\n", + mlflowName: "my-run", + wantVersion: new("6"), wantEnvCell: "6", + wantSource: new("uploaded"), wantRetries: &runMaxRetries{Limit: 1, Source: maxRetriesConfigured}, wantBody: "max_retries: 1", + }, + { + name: "uploaded config without max_retries uses the CLI default", + run: `{"run_id": 1, ` + submitAiRuntime + `}`, + config: "experiment_name: e\n", + wantEnvCell: na, wantSource: new("uploaded"), + wantRetries: &runMaxRetries{Limit: 3, Source: maxRetriesCLIDefault}, wantBody: "experiment_name: e", + }, + { + name: "server-reported unlimited", + run: `{"run_id": 1, "run_type": "SUBMIT_RUN", "tasks": [{"max_retries": -1, "ai_runtime_task": {}}]}`, + wantEnvCell: na, wantRetries: &runMaxRetries{Limit: -1, Source: maxRetriesReported}, + }, + { + name: "job run with a readable config is unknown", + run: `{"run_id": 1, "run_type": "JOB_RUN", "tasks": [{"ai_runtime_task": {}}]}`, + config: "max_retries: 1\n", + wantEnvCell: na, wantSource: new("uploaded"), wantBody: "max_retries: 1", + }, + { + name: "base environment, config unreadable", + run: `{"run_id": 1, "environments": [{"environment_key": "default", "spec": {"base_environment": "workspace-base-environments/databricks_ai_v5"}}], ` + submitAiRuntime + `}`, + wantBase: new("workspace-base-environments/databricks_ai_v5"), wantEnvCell: na, + }, + { + name: "legacy inline parameters", + run: `{"run_id": 1, "run_type": "SUBMIT_RUN", "tasks": [{"gen_ai_compute_task": {"yaml_parameters": "max_retries: 2\ncommand: echo hi\n"}}]}`, + wantEnvCell: na, wantSource: new("yaml_parameters"), + wantRetries: &runMaxRetries{Limit: 2, Source: maxRetriesConfigured}, wantBody: "max_retries: 2\ncommand: echo hi", + }, + { + name: "legacy inline parameters without max_retries are unknown", + run: `{"run_id": 1, "run_type": "SUBMIT_RUN", "tasks": [{"gen_ai_compute_task": {"yaml_parameters": "command: echo hi\n"}}]}`, + wantEnvCell: na, wantSource: new("yaml_parameters"), wantBody: "command: echo hi", + }, + { + name: "legacy synthesized config is not exposed", + run: `{"run_id": 1, "run_type": "SUBMIT_RUN", "tasks": [{"gen_ai_compute_task": {"mlflow_experiment_name": "exp"}}]}`, + wantEnvCell: na, wantBody: "experiment_name: exp", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/api/2.2/jobs/runs/get": + _, _ = w.Write([]byte(tt.run)) + case strings.HasPrefix(r.URL.Path, "/api/2.0/workspace/export") && tt.config != "": + _, _ = w.Write([]byte(tt.config)) + case r.URL.Path == "/api/2.0/mlflow/runs/get" && tt.mlflowName != "": + _, _ = w.Write([]byte(`{"run": {"info": {"run_name": "` + tt.mlflowName + `"}}}`)) + default: + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "not found"}`)) + } + })) + t.Cleanup(srv.Close) + w := newTestWorkspaceClient(t, srv.URL) + ctx := withQuietEnrichment(cmdio.MockDiscard(t.Context())) + run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: 1}) + require.NoError(t, err) + data := buildGetData(run) + if genAIComputeTask(run) == nil { + data.TrainingConfigPath = uploaded + } + + enrichRunDetails(ctx, w, run, &data, &mlflowIdentifiers{ExperimentID: "e1", RunID: "r1"}, true) + + assert.Equal(t, tt.wantVersion, data.EnvironmentVersion) + assert.Equal(t, tt.wantBase, data.BaseEnvironment) + assert.Equal(t, tt.wantEnvCell, data.EnvironmentDisplay) + assert.Equal(t, tt.wantSource, data.ConfigSource) + assert.Equal(t, tt.wantSource == nil, data.Config == nil) + assert.Equal(t, tt.wantRetries, data.MaxRetries) + assert.Equal(t, tt.wantBody, data.configBody) + if tt.mlflowName == "" { + assert.Nil(t, data.MLflowRunName) + } else { + assert.Equal(t, tt.mlflowName, *data.MLflowRunName) + } + + // JSON is null exactly when the text cell is N/A, and both render the + // one resolved value. + body, err := json.Marshal(data) + require.NoError(t, err) + var got struct { + MaxRetries *runMaxRetries `json:"max_retries"` + } + require.NoError(t, json.Unmarshal(body, &got)) + assert.Equal(t, maxRetriesDisplay(data.MaxRetries) == na, got.MaxRetries == nil) + assert.Equal(t, maxRetriesDisplay(data.MaxRetries), maxRetriesDisplay(got.MaxRetries)) + }) + } +} + +func TestEnrichRunDetailsWithoutConfig(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/2.2/jobs/runs/get" { + _, _ = w.Write([]byte(`{"run_id":1}`)) + return + } + if strings.HasPrefix(r.URL.Path, "/api/2.0/workspace/export") { + t.Errorf("unexpected config download: %s", r.URL.Path) + } + w.WriteHeader(http.StatusNotFound) + })) + t.Cleanup(srv.Close) + w := newTestWorkspaceClient(t, srv.URL) + run := &jobs.Run{RunId: 1, RunType: jobs.RunTypeSubmitRun, Tasks: []jobs.RunTask{{AiRuntimeTask: &jobs.AiRuntimeTask{}}}} + data := buildGetData(run) + data.TrainingConfigPath = "/Workspace/run/training_config.yaml" + enrichRunDetails(withQuietEnrichment(t.Context()), w, run, &data, nil, false) + assert.Nil(t, data.Config) + assert.Nil(t, data.ConfigSource) + assert.Nil(t, data.MaxRetries) + + // Inline legacy parameters must not be parsed either without opt-in. + legacy := &jobs.Run{RunId: 1, RunType: jobs.RunTypeSubmitRun, Tasks: []jobs.RunTask{{ + GenAiComputeTask: &jobs.GenAiComputeTask{YamlParameters: "max_retries: 2\n"}, + }}} + legacyData := buildGetData(legacy) + enrichRunDetails(withQuietEnrichment(t.Context()), w, legacy, &legacyData, nil, false) + assert.Nil(t, legacyData.Config) + assert.Nil(t, legacyData.MaxRetries) + + // Server-reported limits remain available without reading config. + legacy.Tasks[0].MaxRetries = 4 + enrichRunDetails(withQuietEnrichment(t.Context()), w, legacy, &legacyData, nil, false) + assert.Equal(t, &runMaxRetries{Limit: 4, Source: maxRetriesReported}, legacyData.MaxRetries) +} + +func TestSweepJSON(t *testing.T) { + exp := "exp" + body, err := json.Marshal(getData{RunID: "1", Sweep: &sweepInfo{ + Total: 2, Succeeded: 1, Active: 1, Completed: 1, + Tasks: []sweepTask{ + {TaskKey: "t0", RunID: "10", Status: "SUCCESS", Experiment: exp, ExperimentName: &exp}, + {TaskKey: "t1", RunID: "11", Status: "RUNNING"}, + }, + }}) + require.NoError(t, err) + var got struct { + Sweep json.RawMessage `json:"sweep"` + } + require.NoError(t, json.Unmarshal(body, &got)) + assert.JSONEq(t, `{"total":2,"succeeded":1,"failed":0,"active":1,"completed":1,"tasks":[ + {"task_key":"t0","run_id":"10","status":"SUCCESS","experiment_name":"exp"}, + {"task_key":"t1","run_id":"11","status":"RUNNING","experiment_name":null}]}`, string(got.Sweep)) +} diff --git a/cmd/air/list_detail_test.go b/cmd/air/list_detail_test.go index 5f880bc1481..05eeb2a6ed2 100644 --- a/cmd/air/list_detail_test.go +++ b/cmd/air/list_detail_test.go @@ -196,3 +196,28 @@ func TestRunDetailTextReadsTheUploadedConfig(t *testing.T) { }) } } + +func TestRunDetailTextShowsEnvironmentAndLegacyConfig(t *testing.T) { + // The list picker's detail pane renders through runDetailText, not `air get`, + // so it must still resolve the environment and the full legacy config. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/2.2/jobs/runs/get": + _, _ = w.Write([]byte(`{"run_id": 123, "run_type": "SUBMIT_RUN", "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "environments": [{"environment_key": "default", "spec": {"environment_version": "4"}}], + "tasks": [{"run_id": 456, "attempt_number": 0, "gen_ai_compute_task": { + "compute": {"gpu_type": "GPU_1xA10", "num_gpus": 1}, + "yaml_parameters": "experiment_name: my-exp\nmax_retries: 2\ncommand: echo legacy-command\n"}}]}`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + })) + t.Cleanup(srv.Close) + + got, err := runDetailText(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL), 123) + require.NoError(t, err) + assert.Contains(t, got, "command: echo legacy-command") + assert.Regexp(t, `Environment\s+4`, got) + assert.Regexp(t, `Max Retries\s+2 `, got) + assert.Regexp(t, `Accelerators\s+1x A10`, got) +} diff --git a/cmd/air/render.go b/cmd/air/render.go index acdaf7f7c78..6830f63680d 100644 --- a/cmd/air/render.go +++ b/cmd/air/render.go @@ -11,9 +11,10 @@ import ( "github.com/charmbracelet/lipgloss" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/httpclient" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/muesli/termenv" "go.yaml.in/yaml/v3" @@ -99,12 +100,11 @@ func renderRunText(ctx context.Context, out io.Writer, w *databricks.WorkspaceCl renderer, colorOn := cmdio.NewRenderer(ctx, out) p := newPalette(renderer) - // The serverless environment version lives on the run's environments[].spec, - // which the typed SDK Run drops, so read it with a raw request. Empty (a run - // with no serverless environment) stays "N/A". - data.EnvironmentDisplay = orNA(aiRuntimeEnvironmentVersion(ctx, w, run.RunId)) - - configBody, configRaw := resolveConfigYAML(ctx, w, run, data) + // The list detail pane (list_detail.go) renders through here without going + // through `air get`, which resolves these up front. + if !data.detailsResolved { + enrichRunDetails(ctx, w, run, data, ids, true) + } view := runView{ runID: data.RunID, @@ -112,7 +112,7 @@ func renderRunText(ctx context.Context, out io.Writer, w *databricks.WorkspaceCl status: data.Status, submitted: data.SubmittedDisplay, retries: data.AttemptNumber, - maxRetries: maxRetriesDisplay(resolveMaxRetries(run, configRaw)), + maxRetries: maxRetriesDisplay(data.MaxRetries), duration: data.DurationDisplay, progress: data.ProgressDisplay, experiment: data.ExperimentDisplay, @@ -126,13 +126,17 @@ func renderRunText(ctx context.Context, out io.Writer, w *databricks.WorkspaceCl } if ids != nil { + name := "" + if data.MLflowRunName != nil { + name = *data.MLflowRunName + } // The box grows to fit its fields, so the label is shown whole. - view.mlflowLabel = mlflowRunLabel(fetchMLflowRunName(ctx, w, ids.RunID), ids.RunID, 0) + ids.retryNote() + view.mlflowLabel = mlflowRunLabel(name, ids.RunID, 0) + ids.retryNote() view.mlflowURL = mlflowRunURL(w.Config.Host, ids) } var sections []string - if body := colorizeConfig(p, configBody); body != "" { + if body := colorizeConfig(p, data.configBody); body != "" { sections = append(sections, renderBox(p, configBoxTitle, body)) } sections = append(sections, renderBox(p, metadataBoxTitle, renderFields(p, colorOn, view))) @@ -166,22 +170,24 @@ func genAIComputeTask(run *jobs.Run) *jobs.GenAiComputeTask { return run.Tasks[0].GenAiComputeTask } -// aiRuntimeEnvironmentVersion returns the serverless environment version an -// ai_runtime run used (e.g. "4"), read from the Jobs GetRun response's -// environments[] entry keyed by aiRuntimeEnvironmentKey. The typed SDK Run has -// no environments field, so the value is fetched with a raw request. Best-effort: -// returns "" (logged) on any error or when the run declares no environment. -func aiRuntimeEnvironmentVersion(ctx context.Context, w *databricks.WorkspaceClient, runID int64) string { +// aiRuntimeEnvironment returns the serverless environment an ai_runtime run used, +// read from the Jobs GetRun response's environments[] entry keyed by +// aiRuntimeEnvironmentKey: a version (e.g. "4") or, for Databricks AI runtimes, a +// base environment. The typed SDK Run has no environments field, so the value is +// fetched with a raw request. Best-effort: returns empty strings (logged) on any +// error or when the run declares no environment. +func aiRuntimeEnvironment(ctx context.Context, w *databricks.WorkspaceClient, runID int64) (version, baseEnvironment string) { apiClient, err := client.New(w.Config) if err != nil { - log.Warnf(ctx, "air get: could not create client to read environment: %v", err) - return "" + enrichmentWarnf(ctx, "air get: could not create client to read environment: %v", err) + return "", "" } var resp struct { Environments []struct { EnvironmentKey string `json:"environment_key"` Spec struct { EnvironmentVersion string `json:"environment_version"` + BaseEnvironment string `json:"base_environment"` } `json:"spec"` } `json:"environments"` } @@ -189,15 +195,15 @@ func aiRuntimeEnvironmentVersion(ctx context.Context, w *databricks.WorkspaceCli // run_id is passed as the request, mirroring the other raw calls in this package. query := map[string]any{"run_id": runID} if err := apiClient.Do(ctx, http.MethodGet, jobsRunsGetPath, auth.WorkspaceIDHeaders(w.Config), nil, query, &resp); err != nil { - log.Warnf(ctx, "air get: could not read environment for run %d: %v", runID, err) - return "" + enrichmentWarnf(ctx, "air get: could not read environment for run %d: %v", runID, err) + return "", "" } for _, e := range resp.Environments { if e.EnvironmentKey == aiRuntimeEnvironmentKey { - return e.Spec.EnvironmentVersion + return e.Spec.EnvironmentVersion, e.Spec.BaseEnvironment } } - return "" + return "", "" } // resolveConfigYAML returns the config box body: from the downloaded config file @@ -232,18 +238,53 @@ func configYAML(ctx context.Context, w *databricks.WorkspaceClient, task *jobs.G return synthConfigYAML(task), nil } -// downloadConfig fetches the run's training-config file, returning nil on -// failure (logged as a warning). Best-effort, like the rest of the enrichment. +// configDownloadTransport limits the body before SDK response handling, which +// otherwise buffers JSON responses and Workspace.Download buffers every response. +type configDownloadTransport struct{ base http.RoundTripper } + +func (t configDownloadTransport) RoundTrip(req *http.Request) (*http.Response, error) { + resp, err := t.base.RoundTrip(req) + if err != nil { + return resp, err + } + resp.Body = struct { + io.Reader + io.Closer + }{io.LimitReader(resp.Body, maxConfigYAMLBytes+1), resp.Body} + return resp, nil +} + +// downloadConfig fetches at most 1 MiB; oversized and unreadable configs are omitted. func downloadConfig(ctx context.Context, w *databricks.WorkspaceClient, path string) []byte { - r, err := w.Workspace.Download(ctx, path) + cfg, err := config.HTTPClientConfigFromConfig(w.Config) if err != nil { - log.Warnf(ctx, "air get: could not download training config %s: %v", path, err) + enrichmentWarnf(ctx, "air get: could not create client to download config: %v", err) + return nil + } + transport := cfg.Transport + if transport == nil { + transport = http.DefaultTransport + } + cfg.Transport = configDownloadTransport{base: transport} + apiClient := httpclient.NewApiClient(cfg) + var r io.ReadCloser + err = apiClient.Do(ctx, http.MethodGet, "/api/2.0/workspace/export", + httpclient.WithRequestHeaders(auth.WorkspaceIDHeaders(w.Config)), + httpclient.WithRequestHeader("Content-Type", "application/json"), + httpclient.WithRequestData(map[string]any{"path": path, "direct_download": true}), + httpclient.WithResponseUnmarshal(&r)) + if err != nil { + enrichmentWarnf(ctx, "air get: could not download training config %s: %v", path, err) return nil } defer r.Close() content, err := io.ReadAll(r) if err != nil { - log.Warnf(ctx, "air get: could not read training config %s: %v", path, err) + enrichmentWarnf(ctx, "air get: could not read training config %s: %v", path, err) + return nil + } + if len(content) > maxConfigYAMLBytes { + enrichmentWarnf(ctx, "air get: training config %s exceeds the %d-byte limit", path, maxConfigYAMLBytes) return nil } return content diff --git a/cmd/air/render_test.go b/cmd/air/render_test.go index cd25947529a..10dd316b69a 100644 --- a/cmd/air/render_test.go +++ b/cmd/air/render_test.go @@ -2,16 +2,18 @@ package aircmd import ( "io" + "net/http" + "net/http/httptest" "strings" "testing" "github.com/charmbracelet/lipgloss" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/muesli/termenv" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" ) // asciiPalette returns a palette whose styles emit no escape codes, so render @@ -21,6 +23,26 @@ func asciiPalette(t *testing.T) palette { return newPalette(r) } +// A local HTTP server exercises the SDK export path without a live workspace. +func configTestClient(t *testing.T, path, content string) *databricks.WorkspaceClient { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/2.0/workspace/export" { + assert.Equal(t, path, r.URL.Query().Get("path")) + assert.Equal(t, "true", r.URL.Query().Get("direct_download")) + assert.Equal(t, "application/json", r.Header.Get("Content-Type")) + body, err := io.ReadAll(r.Body) + assert.NoError(t, err) + assert.Empty(t, body) + _, _ = io.WriteString(w, content) + return + } + _, _ = io.WriteString(w, `{}`) + })) + t.Cleanup(srv.Close) + return newTestWorkspaceClient(t, srv.URL) +} + func TestConfigYAML(t *testing.T) { ctx := t.Context() @@ -36,12 +58,9 @@ func TestConfigYAML(t *testing.T) { }) t.Run("downloads the parameters file when there are no inline parameters", func(t *testing.T) { - m := mocks.NewMockWorkspaceClient(t) - m.GetMockWorkspaceAPI().EXPECT(). - Download(mock.Anything, "/Workspace/cfg.yaml"). - Return(io.NopCloser(strings.NewReader("experiment_name: from-file\n")), nil) + w := configTestClient(t, "/Workspace/cfg.yaml", "experiment_name: from-file\n") task := &jobs.GenAiComputeTask{YamlParametersFilePath: "/Workspace/cfg.yaml"} - got, raw := configYAML(ctx, m.WorkspaceClient, task) + got, raw := configYAML(ctx, w, task) assert.Equal(t, "experiment_name: from-file", got) assert.Equal(t, "experiment_name: from-file\n", string(raw)) }) @@ -57,17 +76,65 @@ func TestConfigYAML(t *testing.T) { }) } +func TestDownloadConfigSizeLimit(t *testing.T) { + for _, tc := range []struct { + name string + size int + wantNil bool + }{ + {"at limit", maxConfigYAMLBytes, false}, + {"above limit", maxConfigYAMLBytes + 1, true}, + } { + t.Run(tc.name, func(t *testing.T) { + w := configTestClient(t, "/Workspace/cfg.yaml", strings.Repeat("a", tc.size)) + got := downloadConfig(withQuietEnrichment(t.Context()), w, "/Workspace/cfg.yaml") + if tc.wantNil { + assert.Nil(t, got) + } else { + assert.Len(t, got, tc.size) + } + }) + } +} + +type configCountingReader struct { + reader io.Reader + bytesRead int +} + +func (r *configCountingReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + r.bytesRead += n + return n, err +} + +func TestDownloadConfigLimitsSDKResponseReads(t *testing.T) { + for _, contentType := range []string{"application/json", "application/octet-stream"} { + t.Run(contentType, func(t *testing.T) { + body := &configCountingReader{reader: strings.NewReader(strings.Repeat("x", 4*maxConfigYAMLBytes))} + // The transport simulates a large workspace export without a live service. + transport := validationRoundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.URL.Path != "/api/2.0/workspace/export" { + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}}, Body: io.NopCloser(strings.NewReader(`{}`)), Request: req}, nil + } + return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{contentType}}, Body: io.NopCloser(body), Request: req}, nil + }) + w := validationTestWorkspaceClientWithTransport(t, transport) + got := downloadConfig(withQuietEnrichment(t.Context()), w, "/Workspace/cfg.yaml") + assert.Nil(t, got) + assert.LessOrEqual(t, body.bytesRead, maxConfigYAMLBytes+1) + }) + } +} + func TestResolveConfigYAML(t *testing.T) { ctx := t.Context() t.Run("downloads the training config path for new runs", func(t *testing.T) { - m := mocks.NewMockWorkspaceClient(t) - m.GetMockWorkspaceAPI().EXPECT(). - Download(mock.Anything, "/Workspace/run/training_config.yaml"). - Return(io.NopCloser(strings.NewReader("experiment_name: from-air\n")), nil) + w := configTestClient(t, "/Workspace/run/training_config.yaml", "experiment_name: from-air\n") run := &jobs.Run{Tasks: []jobs.RunTask{{}}} data := &getData{TrainingConfigPath: "/Workspace/run/training_config.yaml"} - got, raw := resolveConfigYAML(ctx, m.WorkspaceClient, run, data) + got, raw := resolveConfigYAML(ctx, w, run, data) assert.Equal(t, "experiment_name: from-air", got) assert.Equal(t, "experiment_name: from-air\n", string(raw)) }) diff --git a/cmd/air/sweep.go b/cmd/air/sweep.go index b346f43f1b6..ba25114e28e 100644 --- a/cmd/air/sweep.go +++ b/cmd/air/sweep.go @@ -10,22 +10,25 @@ import ( ) // sweepInfo summarizes a "foreach" run, which fans a single config out into many -// iterations (a hyperparameter sweep). It is shown only in text output. +// iterations (a hyperparameter sweep). type sweepInfo struct { - Total int - Succeeded int - Failed int - Active int - Completed int - Tasks []sweepTask + Total int `json:"total"` + Succeeded int `json:"succeeded"` + Failed int `json:"failed"` + Active int `json:"active"` + Completed int `json:"completed"` + Tasks []sweepTask `json:"tasks"` } // sweepTask is one iteration of a sweep. type sweepTask struct { - TaskKey string - RunID string - Status string - Experiment string + TaskKey string `json:"task_key"` + RunID string `json:"run_id"` + Status string `json:"status"` + // ExperimentName is null when the iteration names no experiment; Experiment + // is the same value for the text table, which shows it blank. + ExperimentName *string `json:"experiment_name"` + Experiment string `json:"-"` } // findForEachTask returns the run's foreach task if it has one, or nil. A run is @@ -44,7 +47,7 @@ func findForEachTask(run *jobs.Run) *jobs.RunTask { // iterations require a second lookup. If that lookup fails we still return the // counts (logging the failure) so the user sees the summary. func buildSweepInfo(ctx context.Context, w *databricks.WorkspaceClient, task *jobs.RunTask) *sweepInfo { - info := &sweepInfo{} + info := &sweepInfo{Tasks: []sweepTask{}} if task.ForEachTask.Stats != nil && task.ForEachTask.Stats.TaskRunStats != nil { stats := task.ForEachTask.Stats.TaskRunStats info.Total = stats.TotalIterations @@ -68,7 +71,9 @@ func buildSweepInfo(ctx context.Context, w *databricks.WorkspaceClient, task *jo Status: runStatus(it.State), } if it.GenAiComputeTask != nil && it.GenAiComputeTask.MlflowExperimentName != "" { - row.Experiment = stripExperimentUserPrefix(it.GenAiComputeTask.MlflowExperimentName) + exp := stripExperimentUserPrefix(it.GenAiComputeTask.MlflowExperimentName) + row.Experiment = exp + row.ExperimentName = &exp } info.Tasks = append(info.Tasks, row) } diff --git a/cmd/air/sweep_test.go b/cmd/air/sweep_test.go index 10134c0df42..f5987236380 100644 --- a/cmd/air/sweep_test.go +++ b/cmd/air/sweep_test.go @@ -1,6 +1,7 @@ package aircmd import ( + "encoding/json" "testing" "github.com/databricks/databricks-sdk-go/apierr" @@ -77,5 +78,8 @@ func TestBuildSweepInfo(t *testing.T) { info := buildSweepInfo(ctx, m.WorkspaceClient, sweepTaskFixture()) assert.Equal(t, 4, info.Total) assert.Empty(t, info.Tasks) + encoded, err := json.Marshal(info) + require.NoError(t, err) + assert.JSONEq(t, `{"total":4,"succeeded":1,"failed":1,"active":2,"completed":2,"tasks":[]}`, string(encoded)) }) }