From 12c443a8ace976a28b4b446d28e9bf4e56821336 Mon Sep 17 00:00:00 2001 From: Abdul-Kareem Aliu Date: Wed, 11 Nov 2026 22:18:29 +0000 Subject: [PATCH 1/2] air: Name the YAML path in unknown config field errors Unknown or misspelled keys in an `air run` workload YAML surfaced the raw go-yaml error, which names internal Go types (e.g. "field rot_path not found in type aircmd.codeSourceConfig") and gives no path or suggestion. Walk the parsed YAML against configSchema() before decoding and report every unknown key with its line, dotted YAML path, a did-you-mean suggestion, the valid sibling fields, and the matching `-h config.` pointer. When a key has no close match, note that it may come from newer documentation than the running CLI version supports. KnownFields(true) stays as a backstop. Fixes CNXT-2566 and CNXT-2554. Co-authored-by: Isaac --- .../air/unknown-config-field-errors.md | 1 + cmd/air/runconfig_load.go | 92 ++++++++++++++++++ cmd/air/runconfig_test.go | 94 ++++++++++++++++++- 3 files changed, 183 insertions(+), 4 deletions(-) create mode 100644 .nextchanges/air/unknown-config-field-errors.md diff --git a/.nextchanges/air/unknown-config-field-errors.md b/.nextchanges/air/unknown-config-field-errors.md new file mode 100644 index 00000000000..a44a83efa74 --- /dev/null +++ b/.nextchanges/air/unknown-config-field-errors.md @@ -0,0 +1 @@ +* `air run` now names the YAML path of an unknown config field, suggests the closest field, and notes when a field may need a newer CLI. diff --git a/cmd/air/runconfig_load.go b/cmd/air/runconfig_load.go index 2e05e1b50e6..aac4650024e 100644 --- a/cmd/air/runconfig_load.go +++ b/cmd/air/runconfig_load.go @@ -8,10 +8,12 @@ import ( "io" "os" "regexp" + "slices" "strconv" "strings" "sync" + "github.com/databricks/cli/internal/build" "go.yaml.in/yaml/v3" ) @@ -98,6 +100,9 @@ func readRunConfigDocument(path string) (*yaml.Node, []byte, error) { } func finishRunConfigLoad(path string, document *yaml.Node, validationYAML []byte) (*runConfig, error) { + if err := checkUnknownConfigFields(path, document); err != nil { + return nil, err + } // Validate before stripping comments so YAML errors retain source line numbers. cfg, err := decodeRunConfigReader(bytes.NewReader(validationYAML), path) if err != nil { @@ -125,6 +130,93 @@ func finishRunConfigLoad(path string, document *yaml.Node, validationYAML []byte return cfg, nil } +// unknownConfigKey is a YAML key absent from the run config schema. +type unknownConfigKey struct { + line int + key string + parent configField +} + +// checkUnknownConfigFields reports every YAML key missing from the run config +// schema by its YAML path, so users see the field they wrote rather than the +// strict decoder's Go type names. +func checkUnknownConfigFields(path string, document *yaml.Node) error { + var unknown []unknownConfigKey + collectUnknownConfigKeys(document.Content[0], configSchema(), &unknown) + if len(unknown) == 0 { + return nil + } + + blocks := make([]string, 0, len(unknown)) + suggestUpdate := false + for _, u := range unknown { + names := make([]string, 0, len(u.parent.children)) + for _, c := range u.parent.children { + if !c.hidden { + names = append(names, configLeafName(c.path)) + } + } + slices.Sort(names) + + fieldPath := strings.TrimPrefix(u.parent.path+"."+u.key, configHelpRoot+".") + block := fmt.Sprintf("line %d: unknown field %q", u.line, fieldPath) + if suggestion, ok := closestConfigField(u.key, names); ok { + block += fmt.Sprintf("; did you mean %q?", suggestion) + } else { + suggestUpdate = true + } + block += fmt.Sprintf("\nfields under %q are: %s", u.parent.path, strings.Join(names, ", ")) + // `-h` rejects hidden legacy paths, so only point at documented ones. + if !u.parent.hidden { + block += fmt.Sprintf("\nrun \"databricks air run -h %s\" for details", u.parent.path) + } + blocks = append(blocks, block) + } + + msg := strings.Join(blocks, "\n\n") + if len(blocks) > 1 { + msg = fmt.Sprintf("%d unknown fields\n\n%s", len(blocks), msg) + } + if suggestUpdate { + msg += fmt.Sprintf("\n\nfields from newer documentation may not be supported by this CLI (version %s); update the Databricks CLI (run 'databricks version --check')", build.GetInfo().Version) + } + return fmt.Errorf("invalid config %s: %s", path, msg) +} + +// collectUnknownConfigKeys walks node against schema, appending unknown keys in +// document order. Non-object and free-form fields are left to the decoder. +func collectUnknownConfigKeys(node *yaml.Node, schema configField, unknown *[]unknownConfigKey) { + if node.Kind == yaml.AliasNode { + node = node.Alias + } + if len(schema.children) == 0 || schema.freeForm { + return + } + switch node.Kind { + case yaml.SequenceNode: + for _, item := range node.Content { + collectUnknownConfigKeys(item, schema, unknown) + } + case yaml.MappingNode: + for i := 0; i < len(node.Content); i += 2 { + key, value := node.Content[i], node.Content[i+1] + // The decoder folds "<<: *anchor" into this mapping, so its keys belong to schema. + if key.ShortTag() == "!!merge" { + collectUnknownConfigKeys(value, schema, unknown) + continue + } + child, ok := findConfigChild(schema.children, key.Value) + if !ok { + *unknown = append(*unknown, unknownConfigKey{line: key.Line, key: key.Value, parent: schema}) + continue + } + collectUnknownConfigKeys(value, child, unknown) + } + default: + // A scalar where an object is expected is a type error the decoder reports. + } +} + func codeSourceFieldPresent(document *yaml.Node, field string) bool { codeSource, _ := mappingValue(document.Content[0], "code_source") if codeSource == nil || codeSource.Kind != yaml.MappingNode { diff --git a/cmd/air/runconfig_test.go b/cmd/air/runconfig_test.go index e2d4b47c934..91a0ae2116c 100644 --- a/cmd/air/runconfig_test.go +++ b/cmd/air/runconfig_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "github.com/databricks/cli/internal/build" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/cmdctx" "github.com/stretchr/testify/assert" @@ -137,8 +138,8 @@ func TestLoadRunConfig_CodeSourceErrors(t *testing.T) { {"flattened error path", "code_source:\n root_path: \"\"\n", "code_source.root_path cannot be empty"}, {"nested error path", "code_source:\n snapshot:\n root_path: \"\"\n", "code_source.snapshot.root_path cannot be empty"}, {"null snapshot", "code_source:\n snapshot: null\n", "code_source.snapshot.root_path cannot be empty"}, - {"unknown flattened field", "code_source:\n root_path: src\n bogus: true\n", "field bogus not found"}, - {"unknown nested field", "code_source:\n snapshot:\n root_path: src\n bogus: true\n", "field bogus not found"}, + {"unknown flattened field", "code_source:\n root_path: src\n bogus: true\n", `unknown field "code_source.bogus"`}, + {"unknown nested field", "code_source:\n snapshot:\n root_path: src\n bogus: true\n", `unknown field "code_source.snapshot.bogus"`}, } for _, tt := range tests { @@ -326,6 +327,91 @@ func TestLoadRunConfig_UnknownFieldRejected(t *testing.T) { } } +func TestLoadRunConfig_UnknownFieldMessages(t *testing.T) { + tests := []struct { + name string + yaml string + wantParts []string + denyParts []string + }{ + { + name: "top-level typo", + yaml: minimalConfig + "envronment:\n version: 6\n", + wantParts: []string{ + `line 7: unknown field "envronment"; did you mean "environment"?`, + `fields under "config" are: code_source, command, compute,`, + `run "databricks air run -h config" for details`, + }, + denyParts: []string{"runConfig", "newer documentation"}, + }, + { + name: "nested typo", + yaml: minimalConfig + "code_source:\n rot_path: .\n", + wantParts: []string{ + `line 8: unknown field "code_source.rot_path"; did you mean "root_path"?`, + `fields under "config.code_source" are: git, include_paths, remote_volume, root_path`, + `run "databricks air run -h config.code_source" for details`, + }, + denyParts: []string{"codeSourceConfig", "newer documentation"}, + }, + { + name: "list item typo", + yaml: minimalConfig + "permissions:\n - user_name: a@example.test\n levl: CAN_VIEW\n", + wantParts: []string{`line 9: unknown field "permissions.levl"; did you mean "level"?`}, + }, + { + name: "no close match suggests a newer CLI", + yaml: minimalConfig + "code_source:\n root_path: .\n fancy_new_option: true\n", + wantParts: []string{ + `line 9: unknown field "code_source.fancy_new_option"` + "\n", + "may not be supported by this CLI (version " + build.GetInfo().Version + ")", + "databricks version --check", + }, + denyParts: []string{"did you mean"}, + }, + { + name: "every unknown field is reported", + yaml: "envronment: {}\n" + minimalConfig + "compute2: {}\ncode_source:\n rot_path: .\n", + wantParts: []string{ + ": 3 unknown fields\n", + `line 1: unknown field "envronment"`, + `line 8: unknown field "compute2"`, + `line 10: unknown field "code_source.rot_path"`, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := loadRunConfig(writeConfig(t, tt.yaml)) + require.Error(t, err) + for _, part := range tt.wantParts { + assert.Contains(t, err.Error(), part) + } + for _, part := range tt.denyParts { + assert.NotContains(t, err.Error(), part) + } + }) + } +} + +func TestLoadRunConfig_KnownFieldsAcceptedByUnknownFieldCheck(t *testing.T) { + tests := []struct { + name string + yaml string + }{ + {"hidden legacy code_source keys", "code_source:\n type: snapshot\n snapshot:\n root_path: src\n"}, + {"free-form maps", "parameters:\n lr: {nested: 1}\nenv_variables:\n ANY_NAME: x\n"}, + {"merge key", "permissions:\n - <<: {level: CAN_VIEW}\n user_name: a@example.test\n"}, + {"merge key from alias", "permissions:\n - &grant {level: CAN_VIEW, user_name: a@example.test}\n - <<: *grant\n user_name: b@example.test\n"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := loadRunConfig(writeConfig(t, minimalConfig+tt.yaml)) + require.NoError(t, err) + }) + } +} + func TestLoadRunConfig_ErrorLineNumbersUseSource(t *testing.T) { path := writeConfig(t, `# first comment # second comment @@ -339,12 +425,12 @@ unknown_field: true t.Run("without overrides", func(t *testing.T) { _, err := loadRunConfig(path) - require.ErrorContains(t, err, "line 8: field unknown_field") + require.ErrorContains(t, err, `line 8: unknown field "unknown_field"`) }) t.Run("with overrides", func(t *testing.T) { _, err := loadRunConfigWithOverrides(t.Context(), path, []string{"compute.num_accelerators=2"}) - require.ErrorContains(t, err, "line 8: field unknown_field") + require.ErrorContains(t, err, `line 8: unknown field "unknown_field"`) }) } From 19e6703f2ce13a8e5350dc1da61ff33227404470 Mon Sep 17 00:00:00 2001 From: Abdul-Kareem Aliu Date: Wed, 11 Nov 2026 23:55:37 +0000 Subject: [PATCH 2/2] Add PR link to changelog entry Co-authored-by: Isaac --- .nextchanges/air/unknown-config-field-errors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/air/unknown-config-field-errors.md b/.nextchanges/air/unknown-config-field-errors.md index a44a83efa74..56095bdd493 100644 --- a/.nextchanges/air/unknown-config-field-errors.md +++ b/.nextchanges/air/unknown-config-field-errors.md @@ -1 +1 @@ -* `air run` now names the YAML path of an unknown config field, suggests the closest field, and notes when a field may need a newer CLI. +* `air run` now names the YAML path of an unknown config field, suggests the closest field, and notes when a field may need a newer CLI. ([#7526](https://github.com/databricks/cli/pull/7526))