Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/unknown-config-field-errors.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `air run` now names the YAML path of an unknown config field, suggests the closest field, and notes when a field may need a newer CLI. ([#7528](https://github.com/databricks/cli/pull/7528))
29 changes: 20 additions & 9 deletions cmd/air/runconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -643,6 +643,8 @@ type configField struct {
// freeForm marks a user-keyed map (parameters/secrets/env_variables): no
// children, yet any sub-path into it is valid.
freeForm bool
// isList marks a list field, whose YAML items each match children.
isList bool
children []configField // nil for a leaf
}

Expand Down Expand Up @@ -725,22 +727,26 @@ func cutLast(s, sep string) (before, after string, found bool) {
// unknownConfigFieldError reports an unresolvable segment, naming the valid
// siblings and, when one is close enough, a suggestion.
func unknownConfigFieldError(parent configField, part string, matched []string) error {
names := make([]string, 0, len(parent.children))
for _, c := range parent.children {
if c.hidden {
continue
}
names = append(names, configLeafName(c.path))
}
slices.Sort(names)

names := visibleConfigChildNames(parent)
msg := fmt.Sprintf("unknown config field %q", configHelpRoot+"."+strings.Join(matched, "."))
if suggestion, ok := closestConfigField(part, names); ok {
msg += fmt.Sprintf("; did you mean %q?", suggestion)
}
return fmt.Errorf("%s\n\nfields under %q are: %s", msg, parent.path, strings.Join(names, ", "))
}

// visibleConfigChildNames returns the sorted YAML names of parent's documented children.
func visibleConfigChildNames(parent configField) []string {
names := make([]string, 0, len(parent.children))
for _, c := range parent.children {
if !c.hidden {
names = append(names, configLeafName(c.path))
}
}
slices.Sort(names)
return names
}

// closestConfigField returns the nearest candidate by edit distance, if one is
// close enough to be worth suggesting.
func closestConfigField(name string, candidates []string) (string, bool) {
Expand Down Expand Up @@ -798,13 +804,18 @@ func describeStruct(t reflect.Type, prefix string) []configField {
continue
}

elem := f.Type
for elem.Kind() == reflect.Pointer {
elem = elem.Elem()
}
field := configField{
path: prefix + "." + name,
typeName: configTypeName(f.Type),
required: f.Tag.Get("required"),
help: f.Tag.Get("help"),
hidden: f.Tag.Get("hidden") == "yes",
freeForm: freeFormConfigFields[name],
isList: elem.Kind() == reflect.Slice,
}
if nested := underlyingConfigStruct(f.Type); nested != nil && !field.freeForm {
field.children = describeStruct(nested, field.path)
Expand Down
115 changes: 115 additions & 0 deletions cmd/air/runconfig_load.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"strings"
"sync"

"github.com/databricks/cli/internal/build"
"go.yaml.in/yaml/v3"
)

Expand Down Expand Up @@ -98,6 +99,9 @@ func readRunConfigDocument(path string) (*yaml.Node, []byte, error) {
}

func finishRunConfigLoad(path string, document *yaml.Node, validationYAML []byte) (*runConfig, error) {
if err := checkUnknownConfigFields(path, document); err != nil {
return nil, err
}
// Validate before stripping comments so YAML errors retain source line numbers.
cfg, err := decodeRunConfigReader(bytes.NewReader(validationYAML), path)
if err != nil {
Expand Down Expand Up @@ -125,6 +129,117 @@ func finishRunConfigLoad(path string, document *yaml.Node, validationYAML []byte
return cfg, nil
}

// unknownConfigKey is a YAML key absent from the run config schema.
type unknownConfigKey struct {
line int
key string
parent configField
}

// checkUnknownConfigFields reports every YAML key missing from the run config
// schema by its YAML path, so users see the field they wrote rather than the
// strict decoder's Go type names.
func checkUnknownConfigFields(path string, document *yaml.Node) error {
msg := describeUnknownConfigKeys(collectUnknownConfigKeys(document.Content[0], configSchema(), nil), true)
if msg == "" {
return nil
}
return fmt.Errorf("invalid config %s: %s", path, msg)
}

// describeUnknownConfigKeys renders unknown keys for an error message, or ""
// if there are none. withLines is false for --override values, whose line
// numbers point into the flag rather than the config file.
func describeUnknownConfigKeys(unknown []unknownConfigKey, withLines bool) string {
if len(unknown) == 0 {
return ""
}

blocks := make([]string, 0, len(unknown))
suggestUpdate := false
for _, u := range unknown {
names := visibleConfigChildNames(u.parent)
fieldPath := strings.TrimPrefix(u.parent.path+"."+u.key, configHelpRoot+".")
block := fmt.Sprintf("unknown field %q", fieldPath)
if withLines {
block = fmt.Sprintf("line %d: %s", u.line, block)
}
if suggestion, ok := closestConfigField(u.key, names); ok {
block += fmt.Sprintf("; did you mean %q?", suggestion)
} else {
suggestUpdate = true
}
block += fmt.Sprintf("\nfields under %q are: %s", u.parent.path, strings.Join(names, ", "))
// `-h` rejects hidden legacy paths, so only point at documented ones.
if !u.parent.hidden {
block += fmt.Sprintf("\nrun \"databricks air run -h %s\" for details", u.parent.path)
}
blocks = append(blocks, block)
}

msg := strings.Join(blocks, "\n\n")
if len(blocks) > 1 {
msg = fmt.Sprintf("%d unknown fields\n\n%s", len(blocks), msg)
}
// Some unknown keys are deliberately unsupported (e.g. priority, _bases_),
// so the newer-CLI explanation is only a possibility.
if suggestUpdate {
msg += fmt.Sprintf("\n\nif a field comes from newer documentation, it may need a newer CLI than version %s; run 'databricks version --check'", build.GetInfo().Version)
}
return msg
}

// collectUnknownConfigKeys walks node against schema and returns unknown with
// every unknown key appended. Non-object and free-form fields, and values whose
// shape doesn't match the schema, are left to the decoder's type errors.
func collectUnknownConfigKeys(node *yaml.Node, schema configField, unknown []unknownConfigKey) []unknownConfigKey {
if node.Kind == yaml.AliasNode {
node = node.Alias
}
if len(schema.children) == 0 || schema.freeForm {
return unknown
}
switch node.Kind {
case yaml.SequenceNode:
if !schema.isList {
return unknown
}
item := schema
item.isList = false
for _, n := range node.Content {
unknown = collectUnknownConfigKeys(n, item, unknown)
}
case yaml.MappingNode:
if schema.isList {
return unknown
}
for i := 0; i < len(node.Content); i += 2 {
key, value := node.Content[i], node.Content[i+1]
// The decoder folds "<<: *anchor" (or a list of anchors) into this
// mapping, so the merged keys belong to schema.
if key.ShortTag() == "!!merge" {
sources := []*yaml.Node{value}
if value.Kind == yaml.SequenceNode {
sources = value.Content
}
for _, source := range sources {
unknown = collectUnknownConfigKeys(source, schema, unknown)
}
continue
}
child, ok := findConfigChild(schema.children, key.Value)
if !ok {
unknown = append(unknown, unknownConfigKey{line: key.Line, key: key.Value, parent: schema})
continue
}
unknown = collectUnknownConfigKeys(value, child, unknown)
}
default:
// A scalar where an object is expected is a type error the decoder reports.
}
return unknown
}

func codeSourceFieldPresent(document *yaml.Node, field string) bool {
codeSource, _ := mappingValue(document.Content[0], "code_source")
if codeSource == nil || codeSource.Kind != yaml.MappingNode {
Expand Down
31 changes: 21 additions & 10 deletions cmd/air/runconfig_override.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"slices"
"strings"

"github.com/databricks/cli/libs/cmdio"
Expand Down Expand Up @@ -65,16 +64,8 @@ func checkOverridePath(parts []string, node configField, fullPath string) error
name := parts[0]
child, ok := findConfigChild(node.children, name)
if !ok {
names := make([]string, 0, len(node.children))
for _, c := range node.children {
if c.hidden {
continue
}
names = append(names, configLeafName(c.path))
}
slices.Sort(names)
return fmt.Errorf("invalid --override %q: %q is not a known field; available fields are: %s",
fullPath, name, strings.Join(names, ", "))
fullPath, name, strings.Join(visibleConfigChildNames(node), ", "))
}
if len(parts) == 1 {
return nil
Expand Down Expand Up @@ -103,11 +94,18 @@ func applyOverrides(ctx context.Context, document *yaml.Node, entries []override
return errors.New("run config must be a YAML mapping")
}

schema := configSchema()

for _, e := range entries {
value, err := parseOverrideNode(e.raw)
if err != nil {
return fmt.Errorf("invalid --override %q: cannot parse value %q: %w", e.path, e.raw, err)
}
// Check the value here: once spliced into the file's tree, its keys carry
// line numbers from the override string, not the config file.
if msg := describeUnknownConfigKeys(collectUnknownConfigKeys(value, overrideTargetField(schema, e.path), nil), false); msg != "" {
return fmt.Errorf("invalid --override %q: %s", e.path, msg)
}

parts := strings.Split(e.path, ".")
current := root
Expand Down Expand Up @@ -144,6 +142,19 @@ func applyOverrides(ctx context.Context, document *yaml.Node, entries []override
return nil
}

// overrideTargetField resolves a path already accepted by checkOverridePath.
// Paths into a free-form map resolve to the map, which has no schema below it.
func overrideTargetField(schema configField, path string) configField {
field := schema
for part := range strings.SplitSeq(path, ".") {
if field.freeForm {
break
}
field, _ = findConfigChild(field.children, part)
}
return field
}

func parseOverrideNode(raw string) (*yaml.Node, error) {
var document yaml.Node
if err := yaml.Unmarshal([]byte(raw), &document); err != nil {
Expand Down
Loading
Loading