diff --git a/.nextchanges/air/help-revamp.md b/.nextchanges/air/help-revamp.md new file mode 100644 index 00000000000..e06ff6c81b4 --- /dev/null +++ b/.nextchanges/air/help-revamp.md @@ -0,0 +1 @@ +* Reorganize `databricks air` help: subcommands are grouped by task, `air run` has examples, and YAML fields are listed under headings, with pool fields marked optional and `command`, `compute.num_accelerators`, and `compute.accelerator_type` correctly marked required. ([#7512](https://github.com/databricks/cli/pull/7512)) diff --git a/acceptance/air/config-help/output.txt b/acceptance/air/config-help/output.txt index 75dad55277c..1fae9e919f3 100644 --- a/acceptance/air/config-help/output.txt +++ b/acceptance/air/config-help/output.txt @@ -2,29 +2,35 @@ === command help >>> [CLI] air run --help -Submit a training workload to Databricks serverless GPU compute. +Submit a workload to Databricks serverless GPU compute. -The workload is described by a YAML config file (see --file). +The workload is described by a YAML file (see --file). To look up a YAML +field, pass its path to -h as a separate argument: -To look up a config field, pass its path to -h: - - databricks air run -h config - databricks air run -h config.compute - databricks air run -h config.compute.accelerator_type - -The path must be a separate argument: cobra reserves -h as a boolean, so --h=config.compute and -hconfig.compute are not accepted. + databricks air run -h config all fields + databricks air run -h config.compute one section + databricks air run -h config.compute.accelerator_type one field Usage: databricks air run [flags] +Examples: + # Submit and stream logs until the run finishes + databricks air run -f workload.yaml --watch + + # Validate the config against the workspace without submitting + databricks air run -f workload.yaml --dry-run + + # Try a different GPU without editing the file + databricks air run -f workload.yaml --override compute.accelerator_type=GPU_1xH100 + Flags: - --dry-run Validate the config against the workspace without submitting a run -f, --file string Path to the workload YAML config - -h, --help help for run - --idempotency-key string Return the existing run if this key was already used - --override stringArray Override a YAML field, e.g. compute.num_accelerators=8 (repeatable) --watch Stream logs until the run completes + --override stringArray Override a YAML field, e.g. compute.num_accelerators=8 (repeatable) + --dry-run Validate the config against the workspace without submitting a run + --idempotency-key string Return the existing run if this key was already used + -h, --help help for run Global Flags: --debug enable debug logging @@ -36,40 +42,51 @@ Global Flags: >>> [CLI] air run -h config config - The run YAML schema. Pass a field path for details, e.g. config.compute.accelerator_type. - - AI coding agents can use the databricks-ai-runtime skill to write workload - YAML, submit runs, and monitor progress. + Workload YAML schema. For details on a field, pass its path to -h, for example: - Install the skill locally for your coding agent: + databricks air run -h config.compute.accelerator_type - databricks aitools install --skills-only --skills databricks-ai-runtime + Minimal example: + experiment_name: my-run + compute: + num_accelerators: 1 + accelerator_type: GPU_1xA10 + code_source: + root_path: . + command: python "$CODE_SOURCE_PATH/train.py" - Follow the prompts to choose your coding agent and installation scope. - Use --scope project to install into the current project. - See: https://github.com/databricks/databricks-agent-skills/tree/main/skills/databricks-ai-runtime - - Fields: + Workload: experiment_name (required) Name of the experiment. compute (required) Which accelerators to run on and how many. + command (required unless containers is set) Shell command that starts the workload. environment Python dependencies, runtime version, or a Unity Catalog image for a single-command run. - command Shell command that starts the workload. containers Rank-partitioned containers for heterogeneous workloads. env_variables Plain environment variables, as NAME: value. secrets Environment variables sourced from secrets, as NAME: scope/key. code_source Local code to upload and make available to the run. + parameters Free-form values passed through to the workload. + + Execution: max_retries How many times to retry a failed run. timeout_minutes Wall-clock limit for the run in minutes. idempotency_token Reuse token: a repeat submission with the same token returns the existing run instead of starting another. - parameters Free-form values passed through to the workload. + + MLflow tracking: mlflow_run_name Name for the MLflow run. mlflow_experiment_directory Workspace directory holding the MLflow experiment. mlflow_artifact_location DBFS location where MLflow artifacts are written. + + Access and cost attribution: permissions Who may view or manage the run, as a list of principal plus level grants. - usage_policy_name Usage policy to bill the run to, by name. - usage_policy_id Usage policy to bill the run to, by id. + usage_policy_name Usage policy to attribute the run's usage to, by name. + usage_policy_id Usage policy to attribute the run's usage to, by id. -Use "-h config." for details on a field. +For details on a field, run "databricks air run -h config.". + +Let a coding agent write this YAML for you with the databricks-ai-runtime skill: + databricks aitools install --skills-only --skills databricks-ai-runtime + (add --scope project to install into the current project) + https://github.com/databricks/databricks-agent-skills/tree/main/skills/databricks-ai-runtime === nested object lists its fields @@ -83,12 +100,14 @@ config.compute GPU_1xRTX6000 - 1 accelerator per node Fields: - num_accelerators Total number of GPUs to allocate. - accelerator_type Which accelerator to run on, e.g. GPU_1xA10. + num_accelerators (required) Total number of GPUs to allocate. + accelerator_type (required) Which accelerator to run on, e.g. GPU_1xA10. + + Provisioned capacity (optional): pool_id GPU pool to run on, by id. priority_class Scheduling priority within the pool: BEST_EFFORT (lowest, preemptable) or NORMAL. -Use "-h config.compute." for details on a field. +For details on a field, run "databricks air run -h config.compute.". === new submission fields are documented @@ -101,7 +120,7 @@ config.mlflow_artifact_location >>> [CLI] air run -h config.compute.pool_id config.compute.pool_id - GPU pool to run on, by id. A GPU pool is a pre-provisioned accelerator reservation; contact your Databricks account team to provision one. Must be 1-255 characters. List available pools with 'air pools list'. + GPU pool to run on, by id. A GPU pool is a pre-provisioned accelerator reservation; contact your Databricks account team to provision one. Must be 1-255 characters. List available pools with 'databricks air pools list'. Type: string Required: no @@ -122,7 +141,11 @@ config.environment.dependencies >>> [CLI] air run -h config.environment.version config.environment.version - Client image version to pin (default: 6). See environment version release notes: https://docs.databricks.com/aws/en/release-notes/serverless/environment-version/ + Managed environment to start from, e.g. 6 for the Standard environment or databricks_ai_v6 for the Databricks AI environment (default: 6). Standard includes CUDA but, from version 5, not PyTorch; add it with dependencies. The Databricks AI environment adds PyTorch, Transformers, and other ML libraries. See Docs below for supported versions and package lists. + + Docs: + AWS: https://docs.databricks.com/aws/en/machine-learning/ai-runtime/develop-workloads/environment + Azure: https://learn.microsoft.com/azure/databricks/machine-learning/ai-runtime/develop-workloads/environment Type: string or int Required: no @@ -131,15 +154,19 @@ config.environment.version >>> [CLI] air run -h config.compute.accelerator_type config.compute.accelerator_type - Which accelerator to run on, e.g. GPU_1xA10. See https://docs.databricks.com/aws/en/machine-learning/ai-runtime/cli/yaml-config#reference for the current list of supported GPU types. Matched case-sensitively. + Which accelerator to run on, e.g. GPU_1xA10. Matched case-sensitively. Supported accelerator types: GPU_1xH100 - 1 accelerator per node GPU_8xH100 - 8 accelerators per node GPU_1xRTX6000 - 1 accelerator per node + Docs: + AWS: https://docs.databricks.com/aws/en/machine-learning/ai-runtime/connecting/cli/yaml-config#accelerator-types + Azure: https://learn.microsoft.com/azure/databricks/machine-learning/ai-runtime/connecting/cli/yaml-config#accelerator-types + Type: string - Required: no + Required: yes === code source fields @@ -148,12 +175,12 @@ config.code_source Local code to upload and make available to the run. Fields: - root_path Root of the code source to archive. + root_path (required when code_source is set) Root of the code source to archive. remote_volume Volume to upload the archive to. git Pin the snapshot to a specific git revision. include_paths Restrict the archive to these paths, relative to root_path and without "..". -Use "-h config.code_source." for details on a field. +For details on a field, run "databricks air run -h config.code_source.". === union field @@ -177,10 +204,14 @@ config.code_source.root_path >>> [CLI] air run -h compute.num_accelerators config.compute.num_accelerators - Total number of GPUs to allocate. Must be a positive multiple of the accelerator type's per-node GPU count. See https://docs.databricks.com/aws/en/machine-learning/ai-runtime/cli/yaml-config#reference for supported GPU types. + Total number of GPUs to allocate. Must be a positive multiple of the accelerator type's per-node GPU count. See Docs below for GPUs per node. + + Docs: + AWS: https://docs.databricks.com/aws/en/machine-learning/ai-runtime/connecting/cli/yaml-config#accelerator-types + Azure: https://learn.microsoft.com/azure/databricks/machine-learning/ai-runtime/connecting/cli/yaml-config#accelerator-types Type: int - Required: no + Required: yes === unknown field suggests the near match diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index e4cac86c73d..4f0ee221e55 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -2,22 +2,30 @@ === help >>> [CLI] air --help -Run and manage AI Runtime training workloads on Databricks serverless GPU compute. +Run GPU workloads on Databricks serverless compute. -These commands are under active development. +Describe a workload in a YAML file and submit it with "databricks air run". +Follow a run with "databricks air logs" and "databricks air get", or see your +active runs with "databricks air list". Usage: databricks air [command] -Available Commands: +Run workloads: + run Submit a workload from a YAML config + +Manage runs: cancel Cancel one or more runs - convert-to-dabs Convert an AIR run YAML into a Databricks Asset Bundle - get Show status, configuration, and timing details for a specified run - images Manage container images for AI Runtime - list List active runs for the current user - logs Stream or fetch logs for a run - pools View provisioned GPU pools - run Submit a training workload from a YAML config + get Show a run's status, configuration, and timing + list List your active runs + logs Stream or fetch a run's logs + +Package and deploy: + convert-to-dabs Convert a workload YAML into a Databricks Asset Bundle + images Push container images for custom environments + +Provisioned capacity: + pools View GPU pools provisioned for your workspace Flags: -h, --help help for air @@ -30,6 +38,12 @@ Global Flags: Use "databricks air [command] --help" for more information about a command. +Learn more: + YAML fields and example databricks air run -h config + Quickstart (AWS) https://docs.databricks.com/aws/en/machine-learning/ai-runtime/quickstart + Quickstart (Azure) https://learn.microsoft.com/azure/databricks/machine-learning/ai-runtime/quickstart + Coding agent skill databricks aitools install --skills-only --skills databricks-ai-runtime + === list help >>> [CLI] air list --help @@ -70,9 +84,9 @@ Global Flags: === get help >>> [CLI] air get --help -Show status, configuration, and timing details for a specified run. +Show a run's status, configuration, and timing. -JOB_RUN_ID is the job run ID returned by "air run". +JOB_RUN_ID is the job run ID returned by "databricks air run". Usage: databricks air get JOB_RUN_ID [flags] @@ -89,14 +103,18 @@ Global Flags: === pools help >>> [CLI] air pools --help -View provisioned GPU pools +View GPU pools provisioned for your workspace. + +A GPU pool is dedicated accelerator capacity that your Databricks account team +provisions. Pools are optional: a run uses one only when its YAML sets +compute.pool_id. Usage: databricks air pools [flags] databricks air pools [command] Available Commands: - get Show a GPU pool, including its accelerator usage (POOL_ID may be omitted when the workspace has exactly one pool) + get Show a GPU pool and its accelerator usage list List the GPU pools available to the current workspace Flags: @@ -113,7 +131,7 @@ Use "databricks air pools [command] --help" for more information about a command === images help >>> [CLI] air images --help -Manage container images for AI Runtime +Push container images for custom environments Usage: databricks air images [flags] @@ -173,7 +191,7 @@ Global Flags: Stream logs from an active run until it completes, or fetch logs from a completed run. -JOB_RUN_ID is the job run ID returned by "air run". +JOB_RUN_ID is the job run ID returned by "databricks air run". Usage: databricks air logs JOB_RUN_ID [flags] diff --git a/acceptance/help/output.txt b/acceptance/help/output.txt index 041a319a3d5..cdc5c359bfd 100644 --- a/acceptance/help/output.txt +++ b/acceptance/help/output.txt @@ -181,7 +181,7 @@ Domains domains *Beta* Manage domains for organizing and discovering data assets. Developer Tools - air Run and manage AI Runtime training workloads + air Run and manage AI Runtime workloads on serverless GPU compute bundle Declarative Automation Bundles let you express data/AI/analytics projects as code. sandbox *Beta* Manage Databricks Sandbox environments sync Synchronize a local directory to a workspace directory diff --git a/cmd/air/air.go b/cmd/air/air.go index cfe5d46d175..4154bab891c 100644 --- a/cmd/air/air.go +++ b/cmd/air/air.go @@ -7,31 +7,68 @@ import ( "github.com/spf13/cobra" ) +// Command group IDs for `databricks air --help`, ordered by a run's lifecycle. +const ( + runGroupID = "run" + manageGroupID = "manage" + packageGroupID = "package" + capacityGroupID = "capacity" +) + +const airHelpFooter = ` +Learn more: + YAML fields and example databricks air run -h config + Quickstart (AWS) https://docs.databricks.com/aws/en/machine-learning/ai-runtime/quickstart + Quickstart (Azure) https://learn.microsoft.com/azure/databricks/machine-learning/ai-runtime/quickstart + Coding agent skill databricks aitools install --skills-only --skills databricks-ai-runtime +` + // New returns the root command for the AI Runtime CLI. func New() *cobra.Command { cmd := &cobra.Command{ Use: "air", - Short: "Run and manage AI Runtime training workloads", + Short: "Run and manage AI Runtime workloads on serverless GPU compute", GroupID: "development", - Long: `Run and manage AI Runtime training workloads on Databricks serverless GPU compute. + Long: `Run GPU workloads on Databricks serverless compute. -These commands are under active development.`, +Describe a workload in a YAML file and submit it with "databricks air run". +Follow a run with "databricks air logs" and "databricks air get", or see your +active runs with "databricks air list".`, } + cmd.AddGroup( + &cobra.Group{ID: runGroupID, Title: "Run workloads:"}, + &cobra.Group{ID: manageGroupID, Title: "Manage runs:"}, + &cobra.Group{ID: packageGroupID, Title: "Package and deploy:"}, + &cobra.Group{ID: capacityGroupID, Title: "Provisioned capacity:"}, + ) + runCommand := newRunCommand() wrapRunErrorWithDebugTip(runCommand) - cmd.AddCommand(runCommand) - cmd.AddCommand(newGetCommand()) - cmd.AddCommand(newListCommand()) - cmd.AddCommand(newPoolsCommand()) - cmd.AddCommand(newLogsCommand()) - cmd.AddCommand(newCancelCommand()) - cmd.AddCommand(newConvertToDabsCommand()) - cmd.AddCommand(newImagesCommand()) + addToGroup(cmd, runGroupID, runCommand) + addToGroup(cmd, manageGroupID, newGetCommand(), newListCommand(), newLogsCommand(), newCancelCommand()) + addToGroup(cmd, packageGroupID, newConvertToDabsCommand(), newImagesCommand()) + addToGroup(cmd, capacityGroupID, newPoolsCommand()) + + // Subcommands inherit this help func, so the footer is printed only for `air` itself. + defaultHelp := cmd.HelpFunc() + cmd.SetHelpFunc(func(c *cobra.Command, args []string) { + defaultHelp(c, args) + if c == cmd { + fmt.Fprint(c.OutOrStdout(), airHelpFooter) + } + }) return cmd } +func addToGroup(parent *cobra.Command, groupID string, cmds ...*cobra.Command) { + for _, c := range cmds { + c.GroupID = groupID + parent.AddCommand(c) + } +} + func wrapRunErrorWithDebugTip(cmd *cobra.Command) { runE := cmd.RunE cmd.RunE = func(cmd *cobra.Command, args []string) error { diff --git a/cmd/air/air_test.go b/cmd/air/air_test.go index 10da0fea2a7..11e4546d21d 100644 --- a/cmd/air/air_test.go +++ b/cmd/air/air_test.go @@ -1,7 +1,9 @@ package aircmd import ( + "bytes" "errors" + "strings" "testing" "github.com/spf13/cobra" @@ -27,7 +29,49 @@ func TestNewHelpMetadata(t *testing.T) { cmd := New() assert.Equal(t, "development", cmd.GroupID) - assert.Equal(t, "Run and manage AI Runtime training workloads", cmd.Short) + assert.Equal(t, "Run and manage AI Runtime workloads on serverless GPU compute", cmd.Short) +} + +// TestNewGroupsSubcommands asserts every subcommand is listed under a help +// heading, so none falls into cobra's catch-all "Additional Commands". +func TestNewGroupsSubcommands(t *testing.T) { + cmd := New() + groups := map[string]string{} + for _, g := range cmd.Groups() { + groups[g.ID] = g.Title + } + for _, c := range cmd.Commands() { + assert.Contains(t, groups, c.GroupID, "subcommand %q has no help group", c.Name()) + } + assert.Equal(t, capacityGroupID, findSubcommand(t, cmd, "pools").GroupID) + assert.Equal(t, runGroupID, findSubcommand(t, cmd, "run").GroupID) +} + +func TestHelpFooterOnlyOnAirCommand(t *testing.T) { + cmd := New() + var airOut bytes.Buffer + cmd.SetOut(&airOut) + cmd.HelpFunc()(cmd, nil) + assert.Contains(t, airOut.String(), "Learn more:") + assert.Contains(t, airOut.String(), "databricks air run -h config") + + list := findSubcommand(t, cmd, "list") + var listOut bytes.Buffer + list.SetOut(&listOut) + list.HelpFunc()(list, nil) + assert.True(t, strings.HasPrefix(listOut.String(), "List active runs")) + assert.NotContains(t, listOut.String(), "Learn more:") +} + +func findSubcommand(t *testing.T, parent *cobra.Command, name string) *cobra.Command { + t.Helper() + for _, c := range parent.Commands() { + if c.Name() == name { + return c + } + } + t.Fatalf("subcommand %q not found", name) + return nil } func TestRunErrorIncludesDebugTip(t *testing.T) { diff --git a/cmd/air/compute.go b/cmd/air/compute.go index 03fa6d75a5e..2610f242695 100644 --- a/cmd/air/compute.go +++ b/cmd/air/compute.go @@ -91,10 +91,10 @@ func gpusPerNode(g gpuType) (int, error) { // computeConfig is the `compute` block of the run YAML: which accelerators to // use and how many. type computeConfig struct { - NumAccelerators int `yaml:"num_accelerators" help:"Total number of GPUs to allocate. Must be a positive multiple of the accelerator type's per-node GPU count. See https://docs.databricks.com/aws/en/machine-learning/ai-runtime/cli/yaml-config#reference for supported GPU types."` - AcceleratorType string `yaml:"accelerator_type" help:"Which accelerator to run on, e.g. GPU_1xA10. See https://docs.databricks.com/aws/en/machine-learning/ai-runtime/cli/yaml-config#reference for the current list of supported GPU types. Matched case-sensitively."` - PoolID *string `yaml:"pool_id" help:"GPU pool to run on, by id. A GPU pool is a pre-provisioned accelerator reservation; contact your Databricks account team to provision one. Must be 1-255 characters. List available pools with 'air pools list'."` - PriorityClass *string `yaml:"priority_class" help:"Scheduling priority within the pool: BEST_EFFORT (lowest, preemptable) or NORMAL. Requires pool_id."` + NumAccelerators int `yaml:"num_accelerators" help:"Total number of GPUs to allocate. Must be a positive multiple of the accelerator type's per-node GPU count. See Docs below for GPUs per node." required:"yes" docs:"/machine-learning/ai-runtime/connecting/cli/yaml-config#accelerator-types"` + AcceleratorType string `yaml:"accelerator_type" help:"Which accelerator to run on, e.g. GPU_1xA10. Matched case-sensitively." required:"yes" docs:"/machine-learning/ai-runtime/connecting/cli/yaml-config#accelerator-types"` + PoolID *string `yaml:"pool_id" help:"GPU pool to run on, by id. A GPU pool is a pre-provisioned accelerator reservation; contact your Databricks account team to provision one. Must be 1-255 characters. List available pools with 'databricks air pools list'." group:"Provisioned capacity (optional)"` + PriorityClass *string `yaml:"priority_class" help:"Scheduling priority within the pool: BEST_EFFORT (lowest, preemptable) or NORMAL. Requires pool_id." group:"Provisioned capacity (optional)"` } // validate checks the compute block against the backend's constraints. diff --git a/cmd/air/convert_to_dabs.go b/cmd/air/convert_to_dabs.go index 02b29a4d02f..dcb90cfea00 100644 --- a/cmd/air/convert_to_dabs.go +++ b/cmd/air/convert_to_dabs.go @@ -61,10 +61,10 @@ func newConvertToDabsCommand() *cobra.Command { ) cmd := &cobra.Command{ - Use: "convert-to-dabs ", + Use: "convert-to-dabs YAML_PATH", Args: root.ExactArgs(1), - Short: "Convert an AIR run YAML into a Databricks Asset Bundle", - Long: `Convert an AIR CLI run YAML config into a Databricks Asset Bundle (DABs). + Short: "Convert a workload YAML into a Databricks Asset Bundle", + Long: `Convert an AI Runtime workload YAML into a Databricks Asset Bundle (DABs). The emitted bundle can be deployed with the standard DABs workflow: diff --git a/cmd/air/convert_to_dabs_test.go b/cmd/air/convert_to_dabs_test.go index b3fb98b78ed..ba2c4ce6f71 100644 --- a/cmd/air/convert_to_dabs_test.go +++ b/cmd/air/convert_to_dabs_test.go @@ -78,7 +78,7 @@ code_source: func TestConvertToDabsCommandShape(t *testing.T) { cmd := newConvertToDabsCommand() - assert.Equal(t, "convert-to-dabs ", cmd.Use) + assert.Equal(t, "convert-to-dabs YAML_PATH", cmd.Use) assert.Empty(t, cmd.Commands(), "convert-to-dabs must not register subcommands") // Exactly one positional (the YAML path). assert.NoError(t, cmd.Args(cmd, []string{"run.yaml"})) diff --git a/cmd/air/get.go b/cmd/air/get.go index ddb4b203bef..aa20e3b16a2 100644 --- a/cmd/air/get.go +++ b/cmd/air/get.go @@ -108,10 +108,10 @@ func newGetCommand() *cobra.Command { cmd := &cobra.Command{ Use: "get JOB_RUN_ID", Args: root.ExactArgs(1), - Short: "Show status, configuration, and timing details for a specified run", - Long: `Show status, configuration, and timing details for a specified run. + Short: "Show a run's status, configuration, and timing", + Long: `Show a run's status, configuration, and timing. -JOB_RUN_ID is the job run ID returned by "air run".`, +JOB_RUN_ID is the job run ID returned by "databricks air run".`, Annotations: map[string]string{ "template": getTemplate, }, diff --git a/cmd/air/image.go b/cmd/air/image.go index 4ed01c15b78..94b86e1e3f6 100644 --- a/cmd/air/image.go +++ b/cmd/air/image.go @@ -106,7 +106,7 @@ func registryHostFromDockerConfigureOutput(output string) (string, error) { func newImagesCommand() *cobra.Command { cmd := &cobra.Command{ Use: "images", - Short: "Manage container images for AI Runtime", + Short: "Push container images for custom environments", RunE: root.ReportUnknownSubcommand, } cmd.AddCommand(newImagePushCommand()) diff --git a/cmd/air/list.go b/cmd/air/list.go index b80c644d17e..61cfe3d9b3b 100644 --- a/cmd/air/list.go +++ b/cmd/air/list.go @@ -90,7 +90,7 @@ func newListCommand() *cobra.Command { cmd := &cobra.Command{ Use: "list", Args: root.NoArgs, - Short: "List active runs for the current user", + Short: "List your active runs", Long: `List active runs for the current user. Filters: diff --git a/cmd/air/logs.go b/cmd/air/logs.go index 9689e1e8a4a..949c0e21d36 100644 --- a/cmd/air/logs.go +++ b/cmd/air/logs.go @@ -31,11 +31,11 @@ func newLogsCommand() *cobra.Command { cmd := &cobra.Command{ Use: "logs JOB_RUN_ID", Args: root.ExactArgs(1), - Short: "Stream or fetch logs for a run", + Short: "Stream or fetch a run's logs", Long: `Stream logs from an active run until it completes, or fetch logs from a completed run. -JOB_RUN_ID is the job run ID returned by "air run".`, +JOB_RUN_ID is the job run ID returned by "databricks air run".`, Example: ` databricks air logs 123456789 databricks air logs 123456789 --node 1 --retry 0 databricks air logs 123456789 --download-to ./logs`, diff --git a/cmd/air/pools.go b/cmd/air/pools.go index c6bf6c7d9cb..59a1f507be0 100644 --- a/cmd/air/pools.go +++ b/cmd/air/pools.go @@ -184,8 +184,13 @@ func getPool(ctx context.Context, w *databricks.WorkspaceClient, id string) (*pr func newPoolsCommand() *cobra.Command { cmd := &cobra.Command{ Use: "pools", - Short: "View provisioned GPU pools", - RunE: root.ReportUnknownSubcommand, + Short: "View GPU pools provisioned for your workspace", + Long: `View GPU pools provisioned for your workspace. + +A GPU pool is dedicated accelerator capacity that your Databricks account team +provisions. Pools are optional: a run uses one only when its YAML sets +compute.pool_id.`, + RunE: root.ReportUnknownSubcommand, } cmd.AddCommand(newGetPoolCommand()) @@ -237,7 +242,10 @@ func newGetPoolCommand() *cobra.Command { cmd := &cobra.Command{ Use: "get [POOL_ID]", Args: root.MaximumNArgs(1), - Short: "Show a GPU pool, including its accelerator usage (POOL_ID may be omitted when the workspace has exactly one pool)", + Short: "Show a GPU pool and its accelerator usage", + Long: `Show a GPU pool and its accelerator usage. + +POOL_ID may be omitted when the workspace has exactly one pool.`, } cmd.PreRunE = func(cmd *cobra.Command, args []string) error { diff --git a/cmd/air/run.go b/cmd/air/run.go index 332f1aa1218..b18478368b8 100644 --- a/cmd/air/run.go +++ b/cmd/air/run.go @@ -87,19 +87,23 @@ func newRunCommandWithValidationTimeout(validationTimeout time.Duration) *cobra. cmd := &cobra.Command{ Use: "run", Args: root.NoArgs, - Short: "Submit a training workload from a YAML config", - Long: `Submit a training workload to Databricks serverless GPU compute. + Short: "Submit a workload from a YAML config", + Long: `Submit a workload to Databricks serverless GPU compute. -The workload is described by a YAML config file (see --file). +The workload is described by a YAML file (see --file). To look up a YAML +field, pass its path to -h as a separate argument: -To look up a config field, pass its path to -h: + databricks air run -h config all fields + databricks air run -h config.compute one section + databricks air run -h config.compute.accelerator_type one field`, + Example: ` # Submit and stream logs until the run finishes + databricks air run -f workload.yaml --watch - databricks air run -h config - databricks air run -h config.compute - databricks air run -h config.compute.accelerator_type + # Validate the config against the workspace without submitting + databricks air run -f workload.yaml --dry-run -The path must be a separate argument: cobra reserves -h as a boolean, so --h=config.compute and -hconfig.compute are not accepted.`, + # Try a different GPU without editing the file + databricks air run -f workload.yaml --override compute.accelerator_type=GPU_1xH100`, } // cobra passes -h's positional args to the help func before Args/required-flag @@ -128,6 +132,8 @@ The path must be a separate argument: cobra reserves -h as a boolean, so renderConfigField(c.OutOrStdout(), field, computeOptions) }) + // List flags in declaration order (most used first) rather than alphabetically. + cmd.Flags().SortFlags = false cmd.Flags().StringVarP(&file, "file", "f", "", "Path to the workload YAML config") cmd.Flags().BoolVar(&watch, "watch", false, "Stream logs until the run completes") cmd.Flags().StringArrayVar(&overrides, "override", nil, "Override a YAML field, e.g. compute.num_accelerators=8 (repeatable)") diff --git a/cmd/air/runconfig.go b/cmd/air/runconfig.go index 4bd8cf00bae..499eb997ebf 100644 --- a/cmd/air/runconfig.go +++ b/cmd/air/runconfig.go @@ -1,6 +1,7 @@ package aircmd import ( + "cmp" "errors" "fmt" "io" @@ -53,26 +54,26 @@ var secretTemplateRe = sync.OnceValue(func() *regexp.Regexp { return regexp.Must // runConfig is the top-level run YAML schema: experiment_name + compute / // environment / code_source plus the command and run options. type runConfig struct { - ExperimentName string `yaml:"experiment_name" help:"Name of the experiment. Becomes the Jobs API task key: max 100 characters, alphanumerics, hyphens, and underscores only." required:"yes"` - Compute *computeConfig `yaml:"compute" help:"Which accelerators to run on and how many." required:"yes"` - Environment *environmentConfig `yaml:"environment" help:"Python dependencies, runtime version, or a Unity Catalog image for a single-command run."` - Command *string `yaml:"command" help:"Shell command that starts the workload. Mutually exclusive with containers. Max 1000 lines; move longer logic into a script under code_source."` - Containers []containerConfig `yaml:"containers" help:"Rank-partitioned containers for heterogeneous workloads. Mutually exclusive with command."` - EnvVariables map[string]string `yaml:"env_variables" help:"Plain environment variables, as NAME: value. A name here cannot also appear in secrets."` - Secrets map[string]string `yaml:"secrets" help:"Environment variables sourced from secrets, as NAME: scope/key."` - CodeSource *codeSourceConfig `yaml:"code_source" help:"Local code to upload and make available to the run."` + ExperimentName string `yaml:"experiment_name" help:"Name of the experiment. Becomes the Jobs API task key: max 100 characters, alphanumerics, hyphens, and underscores only." required:"yes" group:"Workload"` + Compute *computeConfig `yaml:"compute" help:"Which accelerators to run on and how many." required:"yes" group:"Workload"` + Environment *environmentConfig `yaml:"environment" help:"Python dependencies, runtime version, or a Unity Catalog image for a single-command run." group:"Workload"` + Command *string `yaml:"command" help:"Shell command that starts the workload. Mutually exclusive with containers. Max 1000 lines; move longer logic into a script under code_source." required:"unless containers is set" group:"Workload"` + Containers []containerConfig `yaml:"containers" help:"Rank-partitioned containers for heterogeneous workloads. Mutually exclusive with command." group:"Workload"` + EnvVariables map[string]string `yaml:"env_variables" help:"Plain environment variables, as NAME: value. A name here cannot also appear in secrets." group:"Workload"` + Secrets map[string]string `yaml:"secrets" help:"Environment variables sourced from secrets, as NAME: scope/key." group:"Workload"` + CodeSource *codeSourceConfig `yaml:"code_source" help:"Local code to upload and make available to the run." group:"Workload"` // MaxRetries defaults to 3 when unset; default-filling is a normalization // concern handled at launch, so a nil pointer is left as-is here. - MaxRetries *int `yaml:"max_retries" help:"How many times to retry a failed run. Must be >= 0. Defaults to 3 when unset."` - TimeoutMinutes *int `yaml:"timeout_minutes" help:"Wall-clock limit for the run in minutes. Must be >= 1."` - IdempotencyToken *string `yaml:"idempotency_token" help:"Reuse token: a repeat submission with the same token returns the existing run instead of starting another. Max 64 characters."` - Parameters map[string]any `yaml:"parameters" help:"Free-form values passed through to the workload. Any nested structure is allowed."` - MLflowRunName *string `yaml:"mlflow_run_name" help:"Name for the MLflow run. Max 100 characters, alphanumerics, hyphens, and underscores only."` - MLflowExperimentDirectory *string `yaml:"mlflow_experiment_directory" help:"Workspace directory holding the MLflow experiment. Must start with /Workspace."` - MLflowArtifactLocation *string `yaml:"mlflow_artifact_location" help:"DBFS location where MLflow artifacts are written. A /Volumes path is normalized to dbfs:/Volumes/... ."` - Permissions []permission `yaml:"permissions" help:"Who may view or manage the run, as a list of principal plus level grants."` - UsagePolicyName *string `yaml:"usage_policy_name" help:"Usage policy to bill the run to, by name. Max 127 characters. Mutually exclusive with usage_policy_id."` - UsagePolicyID *string `yaml:"usage_policy_id" help:"Usage policy to bill the run to, by id. Mutually exclusive with usage_policy_name."` + MaxRetries *int `yaml:"max_retries" help:"How many times to retry a failed run. Must be >= 0. Defaults to 3 when unset." group:"Execution"` + TimeoutMinutes *int `yaml:"timeout_minutes" help:"Wall-clock limit for the run in minutes. Must be >= 1." group:"Execution"` + IdempotencyToken *string `yaml:"idempotency_token" help:"Reuse token: a repeat submission with the same token returns the existing run instead of starting another. Max 64 characters." group:"Execution"` + Parameters map[string]any `yaml:"parameters" help:"Free-form values passed through to the workload. Any nested structure is allowed." group:"Workload"` + MLflowRunName *string `yaml:"mlflow_run_name" help:"Name for the MLflow run. Max 100 characters, alphanumerics, hyphens, and underscores only." group:"MLflow tracking"` + MLflowExperimentDirectory *string `yaml:"mlflow_experiment_directory" help:"Workspace directory holding the MLflow experiment. Must start with /Workspace." group:"MLflow tracking"` + MLflowArtifactLocation *string `yaml:"mlflow_artifact_location" help:"DBFS location where MLflow artifacts are written. A /Volumes path is normalized to dbfs:/Volumes/... ." group:"MLflow tracking"` + Permissions []permission `yaml:"permissions" help:"Who may view or manage the run, as a list of principal plus level grants." group:"Access and cost attribution"` + UsagePolicyName *string `yaml:"usage_policy_name" help:"Usage policy to attribute the run's usage to, by name. Max 127 characters. Mutually exclusive with usage_policy_id." group:"Access and cost attribution"` + UsagePolicyID *string `yaml:"usage_policy_id" help:"Usage policy to attribute the run's usage to, by id. Mutually exclusive with usage_policy_name." group:"Access and cost attribution"` // artifactYAML is the source-derived config serialized for upload after overrides. artifactYAML []byte @@ -352,7 +353,7 @@ func validateSecretRefs(secrets map[string]string) error { // settings. type environmentConfig struct { Dependencies dependencies `yaml:"dependencies" help:"Inline list of packages to install. Not allowed alongside unity_catalog_image."` - Version stringOrInt `yaml:"version" help:"Client image version to pin (default: 6). See environment version release notes: https://docs.databricks.com/aws/en/release-notes/serverless/environment-version/"` + Version stringOrInt `yaml:"version" help:"Managed environment to start from, e.g. 6 for the Standard environment or databricks_ai_v6 for the Databricks AI environment (default: 6). Standard includes CUDA but, from version 5, not PyTorch; add it with dependencies. The Databricks AI environment adds PyTorch, Transformers, and other ML libraries. See Docs below for supported versions and package lists." docs:"/machine-learning/ai-runtime/develop-workloads/environment"` UnityCatalogImage string `yaml:"unity_catalog_image" help:"Unity Catalog custom image to run the workload on, as ..:. Not allowed alongside dependencies or version."` } @@ -639,7 +640,11 @@ type configField struct { typeName string required string help string - hidden bool + // group is the help heading the field is listed under; empty means "Fields". + group string + // docs is a docs page path shared by every cloud; see configDocsBases. + docs string + hidden bool // freeForm marks a user-keyed map (parameters/secrets/env_variables): no // children, yet any sub-path into it is valid. freeForm bool @@ -651,18 +656,18 @@ type configField struct { func configSchema() configField { return configField{ path: configHelpRoot, - help: "The run YAML schema. Pass a field path for details, e.g. " + configHelpRoot + `.compute.accelerator_type. - - AI coding agents can use the databricks-ai-runtime skill to write workload - YAML, submit runs, and monitor progress. - - Install the skill locally for your coding agent: - - databricks aitools install --skills-only --skills databricks-ai-runtime - - Follow the prompts to choose your coding agent and installation scope. - Use --scope project to install into the current project. - See: https://github.com/databricks/databricks-agent-skills/tree/main/skills/databricks-ai-runtime`, + help: `Workload YAML schema. For details on a field, pass its path to -h, for example: + + databricks air run -h config.compute.accelerator_type + + Minimal example: + experiment_name: my-run + compute: + num_accelerators: 1 + accelerator_type: GPU_1xA10 + code_source: + root_path: . + command: python "$CODE_SOURCE_PATH/train.py"`, children: describeStruct(reflect.TypeFor[runConfig](), configHelpRoot), } } @@ -803,6 +808,8 @@ func describeStruct(t reflect.Type, prefix string) []configField { typeName: configTypeName(f.Type), required: f.Tag.Get("required"), help: f.Tag.Get("help"), + group: f.Tag.Get("group"), + docs: f.Tag.Get("docs"), hidden: f.Tag.Get("hidden") == "yes", freeForm: freeFormConfigFields[name], } @@ -875,6 +882,13 @@ func renderConfigField(w io.Writer, f configField, computeOptions []computeOptio } } + if f.docs != "" { + fmt.Fprintln(w, "\n Docs:") + for _, d := range configDocsBases { + fmt.Fprintf(w, " %-6s %s%s\n", d.cloud+":", d.base, f.docs) + } + } + if len(f.children) == 0 { fmt.Fprintf(w, "\n Type: %s\n", f.typeName) required := f.required @@ -892,14 +906,98 @@ func renderConfigField(w io.Writer, f configField, computeOptions []computeOptio } width = max(width, len(configLeafName(c.path))) } - fmt.Fprintf(w, "\n Fields:\n") - for _, c := range f.children { - if c.hidden { - continue + for _, group := range configFieldGroups(f.children) { + title := "Fields" + if group != "" { + title = group } - fmt.Fprintf(w, " %-*s %s\n", width, configLeafName(c.path), configFieldSummary(c)) + fmt.Fprintf(w, "\n %s:\n", title) + for _, c := range groupFieldsRequiredFirst(f.children, group) { + fmt.Fprintf(w, " %-*s %s\n", width, configLeafName(c.path), configFieldSummary(c)) + } + } + fmt.Fprintf(w, "\nFor details on a field, run \"databricks air run -h %s.\".\n", f.path) + if f.path == configHelpRoot { + fmt.Fprint(w, agentSkillHelp) } - fmt.Fprintf(w, "\nUse \"-h %s.\" for details on a field.\n", f.path) +} + +// agentSkillHelp closes `-h config`: users look up the schema while writing +// YAML, which is when an agent skill that writes it for them is most useful. +const agentSkillHelp = ` +Let a coding agent write this YAML for you with the databricks-ai-runtime skill: + databricks aitools install --skills-only --skills databricks-ai-runtime + (add --scope project to install into the current project) + https://github.com/databricks/databricks-agent-skills/tree/main/skills/databricks-ai-runtime +` + +// configDocsBases are the per-cloud docs roots; AI Runtime pages share their +// path below these. GCP has no separate AI Runtime docs, so AWS covers it. +var configDocsBases = []struct{ cloud, base string }{ + {"AWS", "https://docs.databricks.com/aws/en"}, + {"Azure", "https://learn.microsoft.com/azure/databricks"}, +} + +// configFieldGroupOrder lists help headings from most to least commonly needed. +// Ungrouped fields come first; unlisted groups follow in declaration order. +var configFieldGroupOrder = []string{"", "Workload", "Execution", "MLflow tracking", "Access and cost attribution", "Provisioned capacity (optional)"} + +// groupFieldsRequiredFirst returns the visible fields under one heading, with +// required fields first so a reader sees what every config must set. +// Otherwise declaration order is kept. +func groupFieldsRequiredFirst(fields []configField, group string) []configField { + var out []configField + for _, c := range fields { + if !c.hidden && c.group == group { + out = append(out, c) + } + } + slices.SortStableFunc(out, func(a, b configField) int { + return cmp.Compare(requiredRank(a), requiredRank(b)) + }) + return out +} + +func requiredRank(f configField) int { + if requiredMarker(f) != "" { + return 0 + } + return 1 +} + +// requiredMarker labels a field that must be set, either always or under a +// stated condition ("when ..."/"unless ..."). Other required tags describe a +// choice between sibling fields and are explained in the field's own help. +func requiredMarker(f configField) string { + switch { + case f.required == "yes": + return "(required)" + case strings.HasPrefix(f.required, "when "), strings.HasPrefix(f.required, "unless "): + return "(required " + f.required + ")" + } + return "" +} + +// configFieldGroups returns the headings used by the visible fields, in display order. +func configFieldGroups(fields []configField) []string { + var used []string + for _, c := range fields { + if !c.hidden && !slices.Contains(used, c.group) { + used = append(used, c.group) + } + } + var out []string + for _, g := range configFieldGroupOrder { + if slices.Contains(used, g) { + out = append(out, g) + } + } + for _, g := range used { + if !slices.Contains(out, g) { + out = append(out, g) + } + } + return out } func fallbackComputeOptions() []computeOption { @@ -957,8 +1055,8 @@ func launchStageBadge(stage string) string { // first sentence of the help text, annotated when the field is required. func configFieldSummary(f configField) string { summary := firstSentence(f.help) - if f.required == "yes" { - summary = "(required) " + summary + if marker := requiredMarker(f); marker != "" { + summary = marker + " " + summary } return summary } diff --git a/cmd/air/runconfig_test.go b/cmd/air/runconfig_test.go index e2d4b47c934..f9b5db9b0e8 100644 --- a/cmd/air/runconfig_test.go +++ b/cmd/air/runconfig_test.go @@ -651,8 +651,9 @@ func TestResolveConfigField(t *testing.T) { wantType string wantRequired string }{ - {"leaf", "config.compute.accelerator_type", "config.compute.accelerator_type", "string", ""}, - {"bare path", "compute.accelerator_type", "config.compute.accelerator_type", "string", ""}, + {"leaf", "config.compute.accelerator_type", "config.compute.accelerator_type", "string", "yes"}, + {"bare path", "compute.accelerator_type", "config.compute.accelerator_type", "string", "yes"}, + {"conditionally required", "config.command", "config.command", "string", "unless containers is set"}, {"top-level required", "config.experiment_name", "config.experiment_name", "string", "yes"}, {"int leaf", "config.max_retries", "config.max_retries", "int", ""}, {"through a slice", "config.permissions.level", "config.permissions.level", "string", "when a grant is listed"}, @@ -777,7 +778,11 @@ func TestWriteConfigFieldHelp(t *testing.T) { require.NoError(t, writeConfigFieldHelp(&leaf, "config.compute.num_accelerators")) assert.Contains(t, leaf.String(), "config.compute.num_accelerators") assert.Contains(t, leaf.String(), "Type: int") - assert.Contains(t, leaf.String(), "Required: no") + assert.Contains(t, leaf.String(), "Required: yes") + + var optional strings.Builder + require.NoError(t, writeConfigFieldHelp(&optional, "config.timeout_minutes")) + assert.Contains(t, optional.String(), "Required: no") var container strings.Builder require.NoError(t, writeConfigFieldHelp(&container, "config.compute")) @@ -785,21 +790,46 @@ func TestWriteConfigFieldHelp(t *testing.T) { assert.Contains(t, container.String(), "num_accelerators") // A container lists its fields rather than printing a type/required pair. assert.NotContains(t, container.String(), "Required:") - assert.Contains(t, container.String(), `Use "-h config.compute." for details`) + assert.Contains(t, container.String(), `For details on a field, run "databricks air run -h config.compute.".`) var environment strings.Builder require.NoError(t, writeConfigFieldHelp(&environment, "config.environment")) - assert.Contains(t, environment.String(), "version Client image version to pin (default: 6).") + assert.Contains(t, environment.String(), "version Managed environment to start from, e.g. 6 for the Standard environment or databricks_ai_v6 for the Databricks AI environment (default: 6).") var environmentVersion strings.Builder require.NoError(t, writeConfigFieldHelp(&environmentVersion, "config.environment.version")) - assert.Contains(t, environmentVersion.String(), "Client image version to pin (default: 6).") - assert.Contains(t, environmentVersion.String(), "https://docs.databricks.com/aws/en/release-notes/serverless/environment-version/") + assert.Contains(t, environmentVersion.String(), "The Databricks AI environment adds PyTorch, Transformers, and other ML libraries.") + assert.Contains(t, environmentVersion.String(), "https://docs.databricks.com/aws/en/machine-learning/ai-runtime/develop-workloads/environment") - // Required top-level fields are flagged in a listing. + // Top-level fields are listed under headings, most commonly needed first. var root strings.Builder require.NoError(t, writeConfigFieldHelp(&root, "config")) - assert.Contains(t, root.String(), "(required) Which accelerators to run on") + rootHelp := root.String() + // Required fields lead their group even when declared later in the struct. + assert.Contains(t, rootHelp, " Workload:\n experiment_name (required) Name of the experiment.\n compute (required) Which accelerators to run on and how many.\n command (required unless containers is set) Shell command that starts the workload.\n environment ") + headings := []string{" Workload:", " Execution:", " MLflow tracking:", " Access and cost attribution:"} + last := -1 + for _, h := range headings { + i := strings.Index(rootHelp, h) + require.Greater(t, i, last, "heading %q is missing or out of order", h) + last = i + } + assert.NotContains(t, rootHelp, " Fields:", "every top-level field belongs to a group") + assert.Contains(t, rootHelp, " databricks air run -h config.compute.accelerator_type\n", "the example must be a runnable command") + // The agent skill pointer follows the fields, so it doesn't push them down. + assert.Greater(t, strings.Index(rootHelp, "databricks-ai-runtime skill"), strings.Index(rootHelp, "usage_policy_id")) + assert.Contains(t, rootHelp, "databricks aitools install --skills-only --skills databricks-ai-runtime") + + // Pool fields are set apart from the fields every run needs. + var compute strings.Builder + require.NoError(t, writeConfigFieldHelp(&compute, "config.compute")) + fields, capacity, found := strings.Cut(compute.String(), " Provisioned capacity (optional):\n") + require.True(t, found) + assert.Contains(t, fields, " Fields:\n num_accelerators (required) ") + assert.Contains(t, fields, " accelerator_type (required) ") + assert.NotContains(t, fields, "pool_id") + assert.Contains(t, capacity, " pool_id ") + assert.Contains(t, capacity, " priority_class ") require.Error(t, writeConfigFieldHelp(&strings.Builder{}, "config.nope")) } @@ -1049,3 +1079,59 @@ func TestConfigSchemaSharedByHelpAndOverride(t *testing.T) { _, err := resolveConfigField("compute.bogus") require.Error(t, err) } + +func TestRequiredMarker(t *testing.T) { + tests := []struct { + required string + want string + }{ + {"yes", "(required)"}, + {"when code_source is set", "(required when code_source is set)"}, + {"unless containers is set", "(required unless containers is set)"}, + // A choice between siblings is explained in the field's own help. + {"one of branch or commit", ""}, + {"", ""}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, requiredMarker(configField{required: tt.required}), tt.required) + } +} + +func TestGroupFieldsRequiredFirst(t *testing.T) { + fields := []configField{ + {path: "config.a", group: "G"}, + {path: "config.b", group: "G", required: "yes"}, + {path: "config.c", group: "Other", required: "yes"}, + {path: "config.d", group: "G", required: "unless a is set"}, + {path: "config.e", group: "G", hidden: true, required: "yes"}, + {path: "config.f", group: "G"}, + } + var got []string + for _, f := range groupFieldsRequiredFirst(fields, "G") { + got = append(got, configLeafName(f.path)) + } + assert.Equal(t, []string{"b", "d", "a", "f"}, got) +} + +func TestConfigFieldHelpListsDocsForEveryCloud(t *testing.T) { + var out strings.Builder + require.NoError(t, writeConfigFieldHelp(&out, "config.environment.version")) + help := out.String() + assert.Contains(t, help, " Docs:\n"+ + " AWS: https://docs.databricks.com/aws/en/machine-learning/ai-runtime/develop-workloads/environment\n"+ + " Azure: https://learn.microsoft.com/azure/databricks/machine-learning/ai-runtime/develop-workloads/environment\n") + assert.Less(t, strings.Index(help, " Docs:"), strings.Index(help, " Type:")) + + // Inline links would only cover one cloud, so docs links live in the Docs section. + for _, f := range configSchema().children { + assertNoInlineDocsLinks(t, f) + } +} + +func assertNoInlineDocsLinks(t *testing.T, f configField) { + t.Helper() + assert.NotContains(t, f.help, "docs.databricks.com", "%s: use the docs tag instead of an inline link", f.path) + for _, c := range f.children { + assertNoInlineDocsLinks(t, c) + } +}