diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index 37c296e1ffc..4e59f178e13 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -162,8 +162,15 @@ func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan !keepSkippedRunAsChange(fullPath, changeDesc) { continue } + synced, err := applyLifecycle(resourceKey, path, changeDesc) + if err != nil { + return nil, fmt.Errorf("failed to apply lifecycle rules for path %s: %w", path, err) + } + if synced == nil { + continue + } - change, err := convertChangeDesc(fullPath, changeDesc) + change, err := convertChangeDesc(fullPath, synced) if err != nil { return nil, fmt.Errorf("failed to compute config change for path %s: %w", path, err) } diff --git a/bundle/configsync/diff_test.go b/bundle/configsync/diff_test.go index b16885f6b71..817c638b3ff 100644 --- a/bundle/configsync/diff_test.go +++ b/bundle/configsync/diff_test.go @@ -5,6 +5,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/databricks-sdk-go/service/serving" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -298,3 +299,86 @@ func TestExtractChangesSkipsPermissionsAndGrants(t *testing.T) { require.True(t, hasNamedPerms, "a resource named permissions must not be skipped") assert.Contains(t, namedPerms, "description") } + +func TestExtractChangesSkipsLifecycleFields(t *testing.T) { + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.genie_spaces.my_space": { + Changes: deployplan.Changes{ + // OverrideChangeDesc promotes etag drift to an update for deploy. + "etag": {Action: deployplan.Update, Old: "1", Remote: "2"}, + "title": {Action: deployplan.Update, Old: "a", New: "a", Remote: "b"}, + }, + }, + "resources.pipelines.my_pipeline": { + Changes: deployplan.Changes{ + // Not one of the backend default's allowed values, so it is a real edit. + "configuration['spark.sql.ansi.enabled']": {Action: deployplan.Update, Old: "true", Remote: "false"}, + }, + }, + "resources.pipelines.other_pipeline": { + Changes: deployplan.Changes{ + // A remote-only map holding a backend default next to a real addition. + "configuration": {Action: deployplan.Update, Remote: map[string]string{ + "spark.sql.ansi.enabled": "true", + "custom.key": "value", + }}, + }, + }, + "resources.model_serving_endpoints.my_endpoint": { + Changes: deployplan.Changes{ + // GET returns no telemetry_config, so the plan reports the whole + // object, and table_names under it is input_only. + "telemetry_config": { + Action: deployplan.Update, + Old: &serving.TelemetryConfig{TableNames: &serving.UnityCatalogTableNames{LogsTable: "main.default.logs"}}, + New: &serving.TelemetryConfig{TableNames: &serving.UnityCatalogTableNames{LogsTable: "main.default.logs"}}, + }, + }, + }, + "resources.jobs.my_job": { + Changes: deployplan.Changes{ + // A task added remotely keeps its fields, including node_type_id, minus the + // backend-managed ones and defaults pinned to a value. + "tasks[task_key='loop']": {Action: deployplan.Update, Remote: map[string]any{ + "task_key": "loop", + "for_each_task": map[string]any{"task": map[string]any{"new_cluster": map[string]any{ + "spark_version": "16.4.x-scala2.12", + "node_type_id": "i3.xlarge", + "aws_attributes": map[string]any{"availability": "SPOT"}, + }}}, + "run_if": "ALL_SUCCESS", + }}, + // input_only: never returned by GET, so a local edit would be removed. + "trigger.table_update.condition": {Action: deployplan.Update, Old: "ANY_UPDATED", New: "ALL_UPDATED"}, + // Backend default the config no longer sets. + "format": {Action: deployplan.Update, Old: "MULTI_TASK", Remote: "MULTI_TASK"}, + // A principal kind switch must still be synced. + "run_as.service_principal_name": {Action: deployplan.Skip, Reason: deployplan.ReasonBackendDefault, Remote: "sp"}, + }, + }, + }} + + changes, err := ExtractChanges(t.Context(), &bundle.Bundle{}, plan) + require.NoError(t, err) + + assert.Equal(t, ResourceChanges{ + "title": {Operation: OperationReplace, Value: "b"}, + }, changes["resources.genie_spaces.my_space"]) + assert.Equal(t, ResourceChanges{ + "configuration['spark.sql.ansi.enabled']": {Operation: OperationAdd, Value: "false", LocalEdit: true}, + }, changes["resources.pipelines.my_pipeline"]) + assert.Equal(t, ResourceChanges{ + "configuration": {Operation: OperationAdd, Value: map[string]any{"custom.key": "value"}}, + }, changes["resources.pipelines.other_pipeline"]) + assert.NotContains(t, changes, "resources.model_serving_endpoints.my_endpoint") + assert.Equal(t, ResourceChanges{ + "tasks[task_key='loop']": {Operation: OperationAdd, Value: map[string]any{ + "task_key": "loop", + "for_each_task": map[string]any{"task": map[string]any{"new_cluster": map[string]any{ + "spark_version": "16.4.x-scala2.12", + "node_type_id": "i3.xlarge", + }}}, + }}, + "run_as.service_principal_name": {Operation: OperationAdd, Value: "sp"}, + }, changes["resources.jobs.my_job"]) +} diff --git a/bundle/configsync/lifecycle.go b/bundle/configsync/lifecycle.go new file mode 100644 index 00000000000..26e1065c006 --- /dev/null +++ b/bundle/configsync/lifecycle.go @@ -0,0 +1,127 @@ +package configsync + +import ( + "fmt" + "reflect" + "slices" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/libs/structs/structpath" +) + +// applyLifecycle applies the direct engine's lifecycle metadata to a change, whatever +// action the plan gave it, and returns nil when nothing is left to sync. The plan action +// is not enough on its own: it is still an update when the config has an undeployed edit +// to such a field, or when a resource's OverrideChangeDesc promotes it (a genie space etag). +// +// A remote value under ignore_remote_changes (output-only, input-only or backend-managed) +// is never written back, and neither is a backend default the config does not set. When +// the plan reports a whole object (one side is nil), these fields are left out of it, and +// the change is skipped if nothing else differs. Inside a remote-only object, only +// defaults pinned to allowed values are left out: a field like node_type_id may be what +// the user picked in the UI. +func applyLifecycle(resourceKey, fieldPath string, cd *deployplan.ChangeDesc) (*deployplan.ChangeDesc, error) { + path, err := structpath.ParsePath(fieldPath) + if err != nil { + return nil, err + } + resourceType := config.GetResourceTypeFromKey(resourceKey) + backendDefault := cd.New == nil && cd.Remote != nil && !matchPattern(jobRunAsChildPattern, resourceKey+"."+fieldPath) + + var f nestedFieldFilter + for _, rc := range []*dresources.ResourceLifecycleConfig{ + dresources.GetResourceConfig(resourceType), + dresources.GetGeneratedResourceConfig(resourceType), + } { + for _, rule := range rc.IgnoreRemoteChanges { + if path.HasPatternPrefix(rule.Field) { + return nil, nil + } + if isNestedPattern(path, rule.Field) { + f.ignored = append(f.ignored, rule.Field) + } + } + if backendDefault && rc.IsBackendDefault(path, cd.Remote) { + return nil, nil + } + if cd.New == nil { + for _, rule := range rc.BackendDefaults { + if len(rule.Values) > 0 && isNestedPattern(path, rule.Field) { + f.defaults.BackendDefaults = append(f.defaults.BackendDefaults, rule) + } + } + } + } + if len(f.ignored) == 0 && len(f.defaults.BackendDefaults) == 0 { + return cd, nil + } + + newValue, err := normalizeValue(cd.New) + if err != nil { + return nil, fmt.Errorf("failed to normalize config value: %w", err) + } + remoteValue, err := normalizeValue(cd.Remote) + if err != nil { + return nil, fmt.Errorf("failed to normalize remote value: %w", err) + } + newValue = stripFields(path, newValue, f.dropFromConfig) + remoteValue = stripFields(path, remoteValue, f.dropFromRemote) + if reflect.DeepEqual(newValue, remoteValue) { + return nil, nil + } + + stripped := *cd + stripped.New = newValue + stripped.Remote = remoteValue + return &stripped, nil +} + +// isNestedPattern reports whether pattern names a field inside the value at path. +func isNestedPattern(path *structpath.PathNode, pattern *structpath.PatternNode) bool { + return pattern.Len() > path.Len() && path.HasPatternPrefix(pattern.Prefix(path.Len())) +} + +// nestedFieldFilter holds the lifecycle rules that name fields inside a whole-object change. +type nestedFieldFilter struct { + ignored []*structpath.PatternNode + defaults dresources.ResourceLifecycleConfig +} + +func (f *nestedFieldFilter) dropFromConfig(path *structpath.PathNode, _ any) bool { + return slices.ContainsFunc(f.ignored, path.HasPatternPrefix) +} + +func (f *nestedFieldFilter) dropFromRemote(path *structpath.PathNode, value any) bool { + return f.dropFromConfig(path, value) || f.defaults.IsBackendDefault(path, value) +} + +// stripFields drops the fields of value, the normalized value at path, for which drop +// returns true. A map left empty only by dropping fields becomes nil. +func stripFields(path *structpath.PathNode, value any, drop func(*structpath.PathNode, any) bool) any { + switch v := value.(type) { + case map[string]any: + result := make(map[string]any, len(v)) + for key, val := range v { + child := structpath.NewStringKey(path, key) + if drop(child, val) { + continue + } + if s := stripFields(child, val, drop); s != nil || val == nil { + result[key] = s + } + } + if len(result) == 0 && len(v) > 0 { + return nil + } + return result + case []any: + result := make([]any, len(v)) + for i, elem := range v { + result[i] = stripFields(structpath.NewIndex(path, i), elem, drop) + } + return result + } + return value +} diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 43a58291ab2..145402cdef2 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -1189,7 +1189,7 @@ func shouldSkipBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *str if cfg == nil || ch.Old != nil || ch.New != nil || ch.Remote == nil { return "", false } - if matchesAnyBackendDefault(cfg, path, ch.Remote) { + if cfg.IsBackendDefault(path, ch.Remote) { return deployplan.ReasonBackendDefault, true } @@ -1205,47 +1205,13 @@ func shouldSkipBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *str iter := rv.MapRange() for iter.Next() { childPath := structpath.NewBracketString(path, iter.Key().String()) - if !matchesAnyBackendDefault(cfg, childPath, iter.Value().Interface()) { + if !cfg.IsBackendDefault(childPath, iter.Value().Interface()) { return "", false } } return deployplan.ReasonBackendDefault, true } -// matchesAnyBackendDefault reports whether the remote value at path matches any of -// the resource's configured backend-default rules (and the rule's allowed values, -// if specified). -func matchesAnyBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, remote any) bool { - for _, rule := range cfg.BackendDefaults { - if !path.HasPatternPrefix(rule.Field) { - continue - } - if len(rule.Values) == 0 { - return true - } - if matchesAllowedValue(remote, rule.Values) { - return true - } - } - return false -} - -// matchesAllowedValue checks if the remote value matches one of the allowed JSON values. -// Each json.RawMessage is unmarshaled into the same type as remote for comparison. -func matchesAllowedValue(remote any, values []json.RawMessage) bool { - remoteType := reflect.TypeOf(remote) - for _, raw := range values { - candidate := reflect.New(remoteType).Interface() - if err := json.Unmarshal(raw, candidate); err != nil { - continue - } - if structdiff.IsEqual(remote, reflect.ValueOf(candidate).Elem().Interface()) { - return true - } - } - return false -} - func allEmpty(values ...any) bool { for _, v := range values { if v == nil { diff --git a/bundle/direct/dresources/config.go b/bundle/direct/dresources/config.go index 7adcc36086a..b630b7e1176 100644 --- a/bundle/direct/dresources/config.go +++ b/bundle/direct/dresources/config.go @@ -6,9 +6,11 @@ import ( "fmt" "io/fs" "path" + "reflect" "strings" "sync" + "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/cli/libs/structs/structpath" "go.yaml.in/yaml/v3" ) @@ -49,6 +51,39 @@ func (b *BackendDefaultRule) UnmarshalYAML(unmarshal func(any) error) error { return nil } +// IsBackendDefault reports whether remote, the remote value at path, matches any of +// the backend-default rules (and the rule's allowed values, if specified). +func (c *ResourceLifecycleConfig) IsBackendDefault(path *structpath.PathNode, remote any) bool { + for _, rule := range c.BackendDefaults { + if !path.HasPatternPrefix(rule.Field) { + continue + } + if len(rule.Values) == 0 { + return true + } + if matchesAllowedValue(remote, rule.Values) { + return true + } + } + return false +} + +// matchesAllowedValue checks if the remote value matches one of the allowed JSON values. +// Each json.RawMessage is unmarshaled into the same type as remote for comparison. +func matchesAllowedValue(remote any, values []json.RawMessage) bool { + remoteType := reflect.TypeOf(remote) + for _, raw := range values { + candidate := reflect.New(remoteType).Interface() + if err := json.Unmarshal(raw, candidate); err != nil { + continue + } + if structdiff.IsEqual(remote, reflect.ValueOf(candidate).Elem().Interface()) { + return true + } + } + return false +} + // RemoteAdditionRule marks a sub-object whose contents the backend co-owns whenever the // object's WhenSet field is set. Inside such an object, a field the config never declared // coming back set from the remote is an addition by the backend, not drift.