Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/no-builtin-accelerator-types.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Invalid `compute.accelerator_type` errors and `air run -h config.compute` no longer list the CLI's built-in accelerator types, which could differ from the workspace's. ([#7520](https://github.com/databricks/cli/pull/7520))
19 changes: 4 additions & 15 deletions cmd/air/compute.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,26 +18,15 @@ const (
gpuType1xRTX6000 gpuType = "GPU_1xRTX6000"
)

// gpuTypes lists every accelerator type understood by this CLI.
// Workspace availability is enforced by the server.
var gpuTypes = []gpuType{gpuType1xA10, gpuType1xH100, gpuType8xH100, gpuType8xB300, gpuType1xRTX6000}

func validGPUTypesHint() string {
names := make([]string, len(gpuTypes))
for i, g := range gpuTypes {
names[i] = string(g)
}
return "valid types are: " + strings.Join(names, ", ")
}

// parseGPUType resolves a YAML accelerator_type string to a gpuType. The match is
// exact: the server's lookup is case-sensitive.
// exact: the server's lookup is case-sensitive. The error lists no types because
// which ones a workspace can use comes from the server, not this CLI.
func parseGPUType(value string) (gpuType, error) {
switch gpuType(value) {
case gpuType1xA10, gpuType8xH100, gpuType1xH100, gpuType8xB300, gpuType1xRTX6000:
return gpuType(value), nil
}
return "", fmt.Errorf("invalid GPU type %q: %s", value, validGPUTypesHint())
return "", fmt.Errorf("%q is not a recognized accelerator type", value)
}

// priorityClass is the AI Runtime scheduling priority a run is given within its
Expand Down Expand Up @@ -101,7 +90,7 @@ type computeConfig struct {
func (c *computeConfig) validate() error {
g, err := parseGPUType(c.AcceleratorType)
if err != nil {
return fmt.Errorf("compute.accelerator_type: %w", err)
return fmt.Errorf("compute.accelerator_type: %w. Run 'databricks air run -h config.compute' to see the types available in your workspace", err)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we can use -h config.compute.accelerator_type instead of -h config.compute here

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like -h config.compute already shows the workspace's eligible compute options so I figured why not use a shorter command

}

if c.NumAccelerators <= 0 {
Expand Down
5 changes: 4 additions & 1 deletion cmd/air/compute_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package aircmd

import (
"fmt"
"strings"
"testing"

Expand Down Expand Up @@ -35,7 +36,9 @@ func TestParseGPUTypeInvalid(t *testing.T) {
t.Run(in, func(t *testing.T) {
_, err := parseGPUType(in)
require.Error(t, err)
assert.Contains(t, err.Error(), "valid types are")
assert.Equal(t, fmt.Sprintf("%q is not a recognized accelerator type", in), err.Error())
// The CLI must not suggest types: only the workspace knows which it can use.
assert.NotContains(t, err.Error(), "GPU_1xA10")
})
}
}
Expand Down
12 changes: 5 additions & 7 deletions cmd/air/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -276,18 +276,16 @@ func loadComputeOptionsForHelp(cmd *cobra.Command, args []string) []computeOptio

if !cmdctx.HasWorkspaceClient(cmd.Context()) {
if err := root.MustWorkspaceClient(cmd, args); err != nil {
cmd.PrintErrf("Warning: couldn't resolve a workspace to list accelerator types; showing the built-in list instead: %v\n", err)
return fallbackComputeOptions()
cmd.PrintErrf("Warning: couldn't resolve a workspace to list accelerator types: %v\n", err)
return nil
}
}

// On failure the help omits the list: the CLI has no workspace-accurate one to show.
options, err := listWorkspaceComputeOptions(cmd.Context(), cmdctx.WorkspaceClient(cmd.Context()))
if err != nil {
unavailable, retryable := classifyValidationFailure(err)
if !unavailable || retryable {
cmd.PrintErrf("Warning: couldn't fetch the accelerator types available in this workspace; showing the built-in list instead: %v\n", err)
}
return fallbackComputeOptions()
cmd.PrintErrln("Warning: couldn't fetch the accelerator types available in this workspace.")
return nil
}
if options == nil {
return []computeOption{}
Expand Down
19 changes: 0 additions & 19 deletions cmd/air/runconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -902,25 +902,6 @@ func renderConfigField(w io.Writer, f configField, computeOptions []computeOptio
fmt.Fprintf(w, "\nUse \"-h %s.<field>\" for details on a field.\n", f.path)
}

func fallbackComputeOptions() []computeOption {
options := make([]computeOption, 0, len(gpuTypes))
for _, acceleratorType := range gpuTypes {
// RTX6000 availability must be confirmed by the workspace API during rollout.
if acceleratorType == gpuType1xRTX6000 {
continue
}
perNode, err := gpusPerNode(acceleratorType)
if err != nil {
continue
}
options = append(options, computeOption{
HardwareAccelerator: string(acceleratorType),
PerNodeAcceleratorCount: &perNode,
})
}
return options
}

func formatPerNodeAcceleratorCount(option computeOption) string {
count := option.PerNodeAcceleratorCount
if count == nil {
Expand Down
30 changes: 15 additions & 15 deletions cmd/air/runconfig_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,7 @@ func TestLoadRunConfig_Errors(t *testing.T) {
{
"bad gpu type",
"experiment_name: r\ncommand: x\ncompute:\n accelerator_type: a100\n num_accelerators: 1\n",
"invalid GPU type",
`compute.accelerator_type: "a100" is not a recognized accelerator type. Run 'databricks air run -h config.compute'`,
},
{
"num_accelerators not a multiple",
Expand Down Expand Up @@ -805,10 +805,7 @@ func TestWriteConfigFieldHelp(t *testing.T) {
}

func TestRunCommandHelpComputeOptions(t *testing.T) {
builtInOptions := ` GPU_1xA10 - 1 accelerator per node
GPU_1xH100 - 1 accelerator per node
GPU_8xH100 - 8 accelerators per node
GPU_8xB300 - 8 accelerators per node`
// An empty wantOptions means the help must omit the list entirely.
tests := []struct {
name string
statusCode int
Expand All @@ -832,16 +829,15 @@ func TestRunCommandHelpComputeOptions(t *testing.T) {
wantOptions: " No accelerator types are currently available for AI Runtime in this workspace.",
},
{
name: "endpoint unavailable falls back",
name: "endpoint unavailable warns and omits the list",
statusCode: http.StatusNotFound,
response: `{"error_code":"NOT_FOUND","message":"ListComputeOptions is not yet enabled."}`,
wantOptions: builtInOptions,
wantWarning: true,
},
{
name: "unexpected API error warns and falls back",
name: "unexpected API error warns and omits the list",
statusCode: http.StatusInternalServerError,
response: `{"error_code":"INTERNAL_ERROR","message":"Internal error."}`,
wantOptions: builtInOptions,
wantWarning: true,
},
}
Expand All @@ -864,13 +860,17 @@ func TestRunCommandHelpComputeOptions(t *testing.T) {
cmd.SetArgs([]string{"-h", "config.compute"})
require.NoError(t, cmd.Execute())

options, found := strings.CutPrefix(out.String(), "config.compute\n Which accelerators to run on and how many.\n\n Supported accelerator types:\n")
require.True(t, found)
options, _, found = strings.Cut(options, "\n\n Fields:")
require.True(t, found)
assert.Equal(t, tt.wantOptions, options)
if tt.wantOptions == "" {
assert.NotContains(t, out.String(), "Supported accelerator types")
} else {
options, found := strings.CutPrefix(out.String(), "config.compute\n Which accelerators to run on and how many.\n\n Supported accelerator types:\n")
require.True(t, found)
options, _, found = strings.Cut(options, "\n\n Fields:")
require.True(t, found)
assert.Equal(t, tt.wantOptions, options)
}
if tt.wantWarning {
assert.Contains(t, errOut.String(), "showing the built-in list")
assert.Equal(t, "Warning: couldn't fetch the accelerator types available in this workspace.\n", errOut.String())
} else {
assert.Empty(t, errOut.String())
}
Expand Down
Loading