Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/get-json-details.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `air get -o json` includes run details including flat `accelerator_type` and `num_accelerators`. Use `--include-config` to include stored config, which may contain plaintext `env_variables` and parameters; downloads over 1 MiB are omitted, and config-dependent `max_retries` is otherwise null. ([#7624](https://github.com/databricks/cli/pull/7624))
104 changes: 103 additions & 1 deletion acceptance/air/get-ai-runtime/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,48 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1
"dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]",
"mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs",
"est_remaining_seconds": null,
"est_percent_complete": null
"est_percent_complete": null,
"user": "user@example.com",
"accelerator_type": "GPU_1xA10",
"num_accelerators": 1,
"environment_version": "4",
"base_environment": null,
"max_retries": null,
"mlflow_run_name": "my-run",
"sweep": null
}
}

=== get (json, include config)

>>> [CLI] air get 123 -o json --include-config
{
"v": 1,
"ts": "[TIMESTAMP]",
"data": {
"run_id": "123",
"status": "SUCCESS",
"started_at": "[TIMESTAMP]",
"duration_seconds": 12,
"attempt_number": 0,
"experiment_name": "my-exp",
"dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]",
"mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs",
"est_remaining_seconds": null,
"est_percent_complete": null,
"user": "user@example.com",
"accelerator_type": "GPU_1xA10",
"num_accelerators": 1,
"environment_version": "4",
"base_environment": null,
"max_retries": {
"limit": 3,
"source": "cli_default"
},
"mlflow_run_name": "my-run",
"sweep": null,
"config": "experiment_name: my-exp\ncompute:\n accelerator_type: a10\n num_accelerators: 1\ncommand: |-\n for i in $(seq 1 10); do\n echo \"step $i\"\n done\n",
"config_source": "uploaded"
}
}

Expand Down Expand Up @@ -88,3 +129,64 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1

Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]
MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1

=== get (json, max_retries set in the config)

>>> [CLI] air get 123 -o json
{
"v": 1,
"ts": "[TIMESTAMP]",
"data": {
"run_id": "123",
"status": "SUCCESS",
"started_at": "[TIMESTAMP]",
"duration_seconds": 12,
"attempt_number": 0,
"experiment_name": "my-exp",
"dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]",
"mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs",
"est_remaining_seconds": null,
"est_percent_complete": null,
"user": "user@example.com",
"accelerator_type": "GPU_1xA10",
"num_accelerators": 1,
"environment_version": "4",
"base_environment": null,
"max_retries": null,
"mlflow_run_name": "my-run",
"sweep": null
}
}

=== get (json, include config, max_retries set)

>>> [CLI] air get 123 -o json --include-config
{
"v": 1,
"ts": "[TIMESTAMP]",
"data": {
"run_id": "123",
"status": "SUCCESS",
"started_at": "[TIMESTAMP]",
"duration_seconds": 12,
"attempt_number": 0,
"experiment_name": "my-exp",
"dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]",
"mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs",
"est_remaining_seconds": null,
"est_percent_complete": null,
"user": "user@example.com",
"accelerator_type": "GPU_1xA10",
"num_accelerators": 1,
"environment_version": "4",
"base_environment": null,
"max_retries": {
"limit": 1,
"source": "configured"
},
"mlflow_run_name": "my-run",
"sweep": null,
"config": "experiment_name: my-exp\ncompute:\n accelerator_type: a10\n num_accelerators: 1\nmax_retries: 1\ncommand: echo ok\n",
"config_source": "uploaded"
}
}
9 changes: 9 additions & 0 deletions acceptance/air/get-ai-runtime/script
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,18 @@ trace $CLI air get 123
title "get (json)"
trace $CLI air get 123 -o json

title "get (json, include config)"
trace $CLI air get 123 -o json --include-config

# The fake runs/get, like the real one, has no max_retries on the task, so the
# limit comes from the uploaded config: the CLI default above, the set value here.
$CLI workspace import "/Workspace/Users/user@example.com/.air/cli_launch/my-exp/my-exp_abc/training_config.yaml" --file training_config_max_retries.yaml --format AUTO --overwrite &> LOG.import-max-retries

title "get (text, max_retries set in the config)"
trace $CLI air get 123

title "get (json, max_retries set in the config)"
trace $CLI air get 123 -o json

title "get (json, include config, max_retries set)"
trace $CLI air get 123 -o json --include-config
13 changes: 12 additions & 1 deletion acceptance/air/get-pending/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,17 @@
"mlflow_url": null,
"status_message": "Waiting for accelerator compute capacity to become available...",
"est_remaining_seconds": null,
"est_percent_complete": null
"est_percent_complete": null,
"user": "user@example.com",
"accelerator_type": "GPU_1xA10",
"num_accelerators": 1,
"environment_version": null,
"base_environment": null,
"max_retries": {
"limit": 0,
"source": "reported"
},
"mlflow_run_name": null,
"sweep": null
}
}
10 changes: 9 additions & 1 deletion acceptance/air/get/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,15 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1
"dashboard_url": "[DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID]",
"mlflow_url": "[DATABRICKS_URL]/ml/experiments/exp1/runs/run1/logs",
"est_remaining_seconds": null,
"est_percent_complete": null
"est_percent_complete": null,
"user": "user@example.com",
"accelerator_type": "GPU_1xA10",
"num_accelerators": 1,
"environment_version": null,
"base_environment": null,
"max_retries": null,
"mlflow_run_name": "my-run",
"sweep": null
}
}

Expand Down
3 changes: 2 additions & 1 deletion acceptance/air/help/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,8 @@ Usage:
databricks air get JOB_RUN_ID [flags]

Flags:
-h, --help help for get
-h, --help help for get
--include-config Include stored config in JSON (downloads over 1 MiB are omitted; may contain plaintext env_variables and parameters)

Global Flags:
--debug enable debug logging
Expand Down
17 changes: 2 additions & 15 deletions cmd/air/format.go
Original file line number Diff line number Diff line change
Expand Up @@ -326,19 +326,6 @@ func stripExperimentUserPrefix(name string) string {
return name
}

// accelerators returns a short description of the GPUs the run uses, such as
// "8x H100", or an empty string if the run has no GPU compute attached.
func accelerators(run *jobs.Run) string {
if len(run.Tasks) == 0 {
return ""
}
task := run.Tasks[0].GenAiComputeTask
if task == nil || task.Compute == nil {
return ""
}
return acceleratorLabel(task.Compute.GpuType, task.Compute.NumGpus)
}

// acceleratorLabel renders a GPU count and type as "8x H100", "" for none, or
// the count alone ("8x") when the type is unrecognized.
func acceleratorLabel(gpuType string, count int) string {
Expand Down Expand Up @@ -376,8 +363,8 @@ const (
// runMaxRetries is a run's retry limit and where it came from. Limit -1 means
// unlimited.
type runMaxRetries struct {
Limit int
Source maxRetriesSource
Limit int `json:"limit"`
Source maxRetriesSource `json:"source"`
}

// resolveMaxRetries returns the retry limit of the run's latest task, or nil
Expand Down
14 changes: 0 additions & 14 deletions cmd/air/format_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,20 +213,6 @@ func TestReformatYAMLForDisplay(t *testing.T) {
assert.Equal(t, "\tnot: valid: yaml:", reformatYAMLForDisplay([]byte("\tnot: valid: yaml:")))
}

func TestAccelerators(t *testing.T) {
assert.Empty(t, accelerators(&jobs.Run{}))
assert.Empty(t, accelerators(&jobs.Run{Tasks: []jobs.RunTask{{}}}))
assert.Empty(t, accelerators(&jobs.Run{Tasks: []jobs.RunTask{{
GenAiComputeTask: &jobs.GenAiComputeTask{},
}}}))
assert.Empty(t, accelerators(&jobs.Run{Tasks: []jobs.RunTask{{
GenAiComputeTask: &jobs.GenAiComputeTask{Compute: &jobs.ComputeConfig{NumGpus: 0}},
}}}))
assert.Equal(t, "8x H100", accelerators(&jobs.Run{Tasks: []jobs.RunTask{{
GenAiComputeTask: &jobs.GenAiComputeTask{Compute: &jobs.ComputeConfig{NumGpus: 8, GpuType: "GPU_8xH100"}},
}}}))
}

func TestAcceleratorLabel(t *testing.T) {
assert.Empty(t, acceleratorLabel("GPU_8xH100", 0))
assert.Equal(t, "8x H100", acceleratorLabel("GPU_8xH100", 8))
Expand Down
Loading
Loading