diff --git a/.nextchanges/air/unknown-config-field-errors.md b/.nextchanges/air/unknown-config-field-errors.md new file mode 100644 index 00000000000..56095bdd493 --- /dev/null +++ b/.nextchanges/air/unknown-config-field-errors.md @@ -0,0 +1 @@ +* `air run` now names the YAML path of an unknown config field, suggests the closest field, and notes when a field may need a newer CLI. ([#7526](https://github.com/databricks/cli/pull/7526)) diff --git a/cmd/air/runconfig_load.go b/cmd/air/runconfig_load.go index 2e05e1b50e6..aac4650024e 100644 --- a/cmd/air/runconfig_load.go +++ b/cmd/air/runconfig_load.go @@ -8,10 +8,12 @@ import ( "io" "os" "regexp" + "slices" "strconv" "strings" "sync" + "github.com/databricks/cli/internal/build" "go.yaml.in/yaml/v3" ) @@ -98,6 +100,9 @@ func readRunConfigDocument(path string) (*yaml.Node, []byte, error) { } func finishRunConfigLoad(path string, document *yaml.Node, validationYAML []byte) (*runConfig, error) { + if err := checkUnknownConfigFields(path, document); err != nil { + return nil, err + } // Validate before stripping comments so YAML errors retain source line numbers. cfg, err := decodeRunConfigReader(bytes.NewReader(validationYAML), path) if err != nil { @@ -125,6 +130,93 @@ func finishRunConfigLoad(path string, document *yaml.Node, validationYAML []byte return cfg, nil } +// unknownConfigKey is a YAML key absent from the run config schema. +type unknownConfigKey struct { + line int + key string + parent configField +} + +// checkUnknownConfigFields reports every YAML key missing from the run config +// schema by its YAML path, so users see the field they wrote rather than the +// strict decoder's Go type names. +func checkUnknownConfigFields(path string, document *yaml.Node) error { + var unknown []unknownConfigKey + collectUnknownConfigKeys(document.Content[0], configSchema(), &unknown) + if len(unknown) == 0 { + return nil + } + + blocks := make([]string, 0, len(unknown)) + suggestUpdate := false + for _, u := range unknown { + names := make([]string, 0, len(u.parent.children)) + for _, c := range u.parent.children { + if !c.hidden { + names = append(names, configLeafName(c.path)) + } + } + slices.Sort(names) + + fieldPath := strings.TrimPrefix(u.parent.path+"."+u.key, configHelpRoot+".") + block := fmt.Sprintf("line %d: unknown field %q", u.line, fieldPath) + if suggestion, ok := closestConfigField(u.key, names); ok { + block += fmt.Sprintf("; did you mean %q?", suggestion) + } else { + suggestUpdate = true + } + block += fmt.Sprintf("\nfields under %q are: %s", u.parent.path, strings.Join(names, ", ")) + // `-h` rejects hidden legacy paths, so only point at documented ones. + if !u.parent.hidden { + block += fmt.Sprintf("\nrun \"databricks air run -h %s\" for details", u.parent.path) + } + blocks = append(blocks, block) + } + + msg := strings.Join(blocks, "\n\n") + if len(blocks) > 1 { + msg = fmt.Sprintf("%d unknown fields\n\n%s", len(blocks), msg) + } + if suggestUpdate { + msg += fmt.Sprintf("\n\nfields from newer documentation may not be supported by this CLI (version %s); update the Databricks CLI (run 'databricks version --check')", build.GetInfo().Version) + } + return fmt.Errorf("invalid config %s: %s", path, msg) +} + +// collectUnknownConfigKeys walks node against schema, appending unknown keys in +// document order. Non-object and free-form fields are left to the decoder. +func collectUnknownConfigKeys(node *yaml.Node, schema configField, unknown *[]unknownConfigKey) { + if node.Kind == yaml.AliasNode { + node = node.Alias + } + if len(schema.children) == 0 || schema.freeForm { + return + } + switch node.Kind { + case yaml.SequenceNode: + for _, item := range node.Content { + collectUnknownConfigKeys(item, schema, unknown) + } + case yaml.MappingNode: + for i := 0; i < len(node.Content); i += 2 { + key, value := node.Content[i], node.Content[i+1] + // The decoder folds "<<: *anchor" into this mapping, so its keys belong to schema. + if key.ShortTag() == "!!merge" { + collectUnknownConfigKeys(value, schema, unknown) + continue + } + child, ok := findConfigChild(schema.children, key.Value) + if !ok { + *unknown = append(*unknown, unknownConfigKey{line: key.Line, key: key.Value, parent: schema}) + continue + } + collectUnknownConfigKeys(value, child, unknown) + } + default: + // A scalar where an object is expected is a type error the decoder reports. + } +} + func codeSourceFieldPresent(document *yaml.Node, field string) bool { codeSource, _ := mappingValue(document.Content[0], "code_source") if codeSource == nil || codeSource.Kind != yaml.MappingNode { diff --git a/cmd/air/runconfig_test.go b/cmd/air/runconfig_test.go index e2d4b47c934..91a0ae2116c 100644 --- a/cmd/air/runconfig_test.go +++ b/cmd/air/runconfig_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "github.com/databricks/cli/internal/build" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/cmdctx" "github.com/stretchr/testify/assert" @@ -137,8 +138,8 @@ func TestLoadRunConfig_CodeSourceErrors(t *testing.T) { {"flattened error path", "code_source:\n root_path: \"\"\n", "code_source.root_path cannot be empty"}, {"nested error path", "code_source:\n snapshot:\n root_path: \"\"\n", "code_source.snapshot.root_path cannot be empty"}, {"null snapshot", "code_source:\n snapshot: null\n", "code_source.snapshot.root_path cannot be empty"}, - {"unknown flattened field", "code_source:\n root_path: src\n bogus: true\n", "field bogus not found"}, - {"unknown nested field", "code_source:\n snapshot:\n root_path: src\n bogus: true\n", "field bogus not found"}, + {"unknown flattened field", "code_source:\n root_path: src\n bogus: true\n", `unknown field "code_source.bogus"`}, + {"unknown nested field", "code_source:\n snapshot:\n root_path: src\n bogus: true\n", `unknown field "code_source.snapshot.bogus"`}, } for _, tt := range tests { @@ -326,6 +327,91 @@ func TestLoadRunConfig_UnknownFieldRejected(t *testing.T) { } } +func TestLoadRunConfig_UnknownFieldMessages(t *testing.T) { + tests := []struct { + name string + yaml string + wantParts []string + denyParts []string + }{ + { + name: "top-level typo", + yaml: minimalConfig + "envronment:\n version: 6\n", + wantParts: []string{ + `line 7: unknown field "envronment"; did you mean "environment"?`, + `fields under "config" are: code_source, command, compute,`, + `run "databricks air run -h config" for details`, + }, + denyParts: []string{"runConfig", "newer documentation"}, + }, + { + name: "nested typo", + yaml: minimalConfig + "code_source:\n rot_path: .\n", + wantParts: []string{ + `line 8: unknown field "code_source.rot_path"; did you mean "root_path"?`, + `fields under "config.code_source" are: git, include_paths, remote_volume, root_path`, + `run "databricks air run -h config.code_source" for details`, + }, + denyParts: []string{"codeSourceConfig", "newer documentation"}, + }, + { + name: "list item typo", + yaml: minimalConfig + "permissions:\n - user_name: a@example.test\n levl: CAN_VIEW\n", + wantParts: []string{`line 9: unknown field "permissions.levl"; did you mean "level"?`}, + }, + { + name: "no close match suggests a newer CLI", + yaml: minimalConfig + "code_source:\n root_path: .\n fancy_new_option: true\n", + wantParts: []string{ + `line 9: unknown field "code_source.fancy_new_option"` + "\n", + "may not be supported by this CLI (version " + build.GetInfo().Version + ")", + "databricks version --check", + }, + denyParts: []string{"did you mean"}, + }, + { + name: "every unknown field is reported", + yaml: "envronment: {}\n" + minimalConfig + "compute2: {}\ncode_source:\n rot_path: .\n", + wantParts: []string{ + ": 3 unknown fields\n", + `line 1: unknown field "envronment"`, + `line 8: unknown field "compute2"`, + `line 10: unknown field "code_source.rot_path"`, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := loadRunConfig(writeConfig(t, tt.yaml)) + require.Error(t, err) + for _, part := range tt.wantParts { + assert.Contains(t, err.Error(), part) + } + for _, part := range tt.denyParts { + assert.NotContains(t, err.Error(), part) + } + }) + } +} + +func TestLoadRunConfig_KnownFieldsAcceptedByUnknownFieldCheck(t *testing.T) { + tests := []struct { + name string + yaml string + }{ + {"hidden legacy code_source keys", "code_source:\n type: snapshot\n snapshot:\n root_path: src\n"}, + {"free-form maps", "parameters:\n lr: {nested: 1}\nenv_variables:\n ANY_NAME: x\n"}, + {"merge key", "permissions:\n - <<: {level: CAN_VIEW}\n user_name: a@example.test\n"}, + {"merge key from alias", "permissions:\n - &grant {level: CAN_VIEW, user_name: a@example.test}\n - <<: *grant\n user_name: b@example.test\n"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := loadRunConfig(writeConfig(t, minimalConfig+tt.yaml)) + require.NoError(t, err) + }) + } +} + func TestLoadRunConfig_ErrorLineNumbersUseSource(t *testing.T) { path := writeConfig(t, `# first comment # second comment @@ -339,12 +425,12 @@ unknown_field: true t.Run("without overrides", func(t *testing.T) { _, err := loadRunConfig(path) - require.ErrorContains(t, err, "line 8: field unknown_field") + require.ErrorContains(t, err, `line 8: unknown field "unknown_field"`) }) t.Run("with overrides", func(t *testing.T) { _, err := loadRunConfigWithOverrides(t.Context(), path, []string{"compute.num_accelerators=2"}) - require.ErrorContains(t, err, "line 8: field unknown_field") + require.ErrorContains(t, err, `line 8: unknown field "unknown_field"`) }) }