Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion bundle/configsync/diff.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,8 +162,15 @@ func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan
!keepSkippedRunAsChange(fullPath, changeDesc) {
continue
}
synced, err := applyLifecycle(resourceKey, path, changeDesc)
if err != nil {
return nil, fmt.Errorf("failed to apply lifecycle rules for path %s: %w", path, err)
}
if synced == nil {
continue
}

change, err := convertChangeDesc(fullPath, changeDesc)
change, err := convertChangeDesc(fullPath, synced)
if err != nil {
return nil, fmt.Errorf("failed to compute config change for path %s: %w", path, err)
}
Expand Down
84 changes: 84 additions & 0 deletions bundle/configsync/diff_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/deployplan"
"github.com/databricks/databricks-sdk-go/service/serving"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
Expand Down Expand Up @@ -298,3 +299,86 @@ func TestExtractChangesSkipsPermissionsAndGrants(t *testing.T) {
require.True(t, hasNamedPerms, "a resource named permissions must not be skipped")
assert.Contains(t, namedPerms, "description")
}

func TestExtractChangesSkipsLifecycleFields(t *testing.T) {
plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{
"resources.genie_spaces.my_space": {
Changes: deployplan.Changes{
// OverrideChangeDesc promotes etag drift to an update for deploy.
"etag": {Action: deployplan.Update, Old: "1", Remote: "2"},
"title": {Action: deployplan.Update, Old: "a", New: "a", Remote: "b"},
},
},
"resources.pipelines.my_pipeline": {
Changes: deployplan.Changes{
// Not one of the backend default's allowed values, so it is a real edit.
"configuration['spark.sql.ansi.enabled']": {Action: deployplan.Update, Old: "true", Remote: "false"},
},
},
"resources.pipelines.other_pipeline": {
Changes: deployplan.Changes{
// A remote-only map holding a backend default next to a real addition.
"configuration": {Action: deployplan.Update, Remote: map[string]string{
"spark.sql.ansi.enabled": "true",
"custom.key": "value",
}},
},
},
"resources.model_serving_endpoints.my_endpoint": {
Changes: deployplan.Changes{
// GET returns no telemetry_config, so the plan reports the whole
// object, and table_names under it is input_only.
"telemetry_config": {
Action: deployplan.Update,
Old: &serving.TelemetryConfig{TableNames: &serving.UnityCatalogTableNames{LogsTable: "main.default.logs"}},
New: &serving.TelemetryConfig{TableNames: &serving.UnityCatalogTableNames{LogsTable: "main.default.logs"}},
},
},
},
"resources.jobs.my_job": {
Changes: deployplan.Changes{
// A task added remotely keeps its fields, including node_type_id, minus the
// backend-managed ones and defaults pinned to a value.
"tasks[task_key='loop']": {Action: deployplan.Update, Remote: map[string]any{
"task_key": "loop",
"for_each_task": map[string]any{"task": map[string]any{"new_cluster": map[string]any{
"spark_version": "16.4.x-scala2.12",
"node_type_id": "i3.xlarge",
"aws_attributes": map[string]any{"availability": "SPOT"},
}}},
"run_if": "ALL_SUCCESS",
}},
// input_only: never returned by GET, so a local edit would be removed.
"trigger.table_update.condition": {Action: deployplan.Update, Old: "ANY_UPDATED", New: "ALL_UPDATED"},
// Backend default the config no longer sets.
"format": {Action: deployplan.Update, Old: "MULTI_TASK", Remote: "MULTI_TASK"},
// A principal kind switch must still be synced.
"run_as.service_principal_name": {Action: deployplan.Skip, Reason: deployplan.ReasonBackendDefault, Remote: "sp"},
},
},
}}

changes, err := ExtractChanges(t.Context(), &bundle.Bundle{}, plan)
require.NoError(t, err)

assert.Equal(t, ResourceChanges{
"title": {Operation: OperationReplace, Value: "b"},
}, changes["resources.genie_spaces.my_space"])
assert.Equal(t, ResourceChanges{
"configuration['spark.sql.ansi.enabled']": {Operation: OperationAdd, Value: "false", LocalEdit: true},
}, changes["resources.pipelines.my_pipeline"])
assert.Equal(t, ResourceChanges{
"configuration": {Operation: OperationAdd, Value: map[string]any{"custom.key": "value"}},
}, changes["resources.pipelines.other_pipeline"])
assert.NotContains(t, changes, "resources.model_serving_endpoints.my_endpoint")
assert.Equal(t, ResourceChanges{
"tasks[task_key='loop']": {Operation: OperationAdd, Value: map[string]any{
"task_key": "loop",
"for_each_task": map[string]any{"task": map[string]any{"new_cluster": map[string]any{
"spark_version": "16.4.x-scala2.12",
"node_type_id": "i3.xlarge",
}}},
}},
"run_as.service_principal_name": {Operation: OperationAdd, Value: "sp"},
}, changes["resources.jobs.my_job"])
}
127 changes: 127 additions & 0 deletions bundle/configsync/lifecycle.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
package configsync

import (
"fmt"
"reflect"
"slices"

"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/deployplan"
"github.com/databricks/cli/bundle/direct/dresources"
"github.com/databricks/cli/libs/structs/structpath"
)

// applyLifecycle applies the direct engine's lifecycle metadata to a change, whatever
// action the plan gave it, and returns nil when nothing is left to sync. The plan action
// is not enough on its own: it is still an update when the config has an undeployed edit
// to such a field, or when a resource's OverrideChangeDesc promotes it (a genie space etag).
//
// A remote value under ignore_remote_changes (output-only, input-only or backend-managed)
// is never written back, and neither is a backend default the config does not set. When
// the plan reports a whole object (one side is nil), these fields are left out of it, and
// the change is skipped if nothing else differs. Inside a remote-only object, only
// defaults pinned to allowed values are left out: a field like node_type_id may be what
// the user picked in the UI.
func applyLifecycle(resourceKey, fieldPath string, cd *deployplan.ChangeDesc) (*deployplan.ChangeDesc, error) {
path, err := structpath.ParsePath(fieldPath)
if err != nil {
return nil, err
}
resourceType := config.GetResourceTypeFromKey(resourceKey)
backendDefault := cd.New == nil && cd.Remote != nil && !matchPattern(jobRunAsChildPattern, resourceKey+"."+fieldPath)

var f nestedFieldFilter
for _, rc := range []*dresources.ResourceLifecycleConfig{
dresources.GetResourceConfig(resourceType),
dresources.GetGeneratedResourceConfig(resourceType),
} {
for _, rule := range rc.IgnoreRemoteChanges {
if path.HasPatternPrefix(rule.Field) {
return nil, nil
}
if isNestedPattern(path, rule.Field) {
f.ignored = append(f.ignored, rule.Field)
}
}
if backendDefault && rc.IsBackendDefault(path, cd.Remote) {
return nil, nil
}
if cd.New == nil {
for _, rule := range rc.BackendDefaults {
if len(rule.Values) > 0 && isNestedPattern(path, rule.Field) {
f.defaults.BackendDefaults = append(f.defaults.BackendDefaults, rule)
}
}
}
}
if len(f.ignored) == 0 && len(f.defaults.BackendDefaults) == 0 {
return cd, nil
}

newValue, err := normalizeValue(cd.New)
if err != nil {
return nil, fmt.Errorf("failed to normalize config value: %w", err)
}
remoteValue, err := normalizeValue(cd.Remote)
if err != nil {
return nil, fmt.Errorf("failed to normalize remote value: %w", err)
}
newValue = stripFields(path, newValue, f.dropFromConfig)
remoteValue = stripFields(path, remoteValue, f.dropFromRemote)
if reflect.DeepEqual(newValue, remoteValue) {
return nil, nil
}

stripped := *cd
stripped.New = newValue
stripped.Remote = remoteValue
return &stripped, nil
}

// isNestedPattern reports whether pattern names a field inside the value at path.
func isNestedPattern(path *structpath.PathNode, pattern *structpath.PatternNode) bool {
return pattern.Len() > path.Len() && path.HasPatternPrefix(pattern.Prefix(path.Len()))
}

// nestedFieldFilter holds the lifecycle rules that name fields inside a whole-object change.
type nestedFieldFilter struct {
ignored []*structpath.PatternNode
defaults dresources.ResourceLifecycleConfig
}

func (f *nestedFieldFilter) dropFromConfig(path *structpath.PathNode, _ any) bool {
return slices.ContainsFunc(f.ignored, path.HasPatternPrefix)
}

func (f *nestedFieldFilter) dropFromRemote(path *structpath.PathNode, value any) bool {
return f.dropFromConfig(path, value) || f.defaults.IsBackendDefault(path, value)
}

// stripFields drops the fields of value, the normalized value at path, for which drop
// returns true. A map left empty only by dropping fields becomes nil.
func stripFields(path *structpath.PathNode, value any, drop func(*structpath.PathNode, any) bool) any {
switch v := value.(type) {
case map[string]any:
result := make(map[string]any, len(v))
for key, val := range v {
child := structpath.NewStringKey(path, key)
if drop(child, val) {
continue
}
if s := stripFields(child, val, drop); s != nil || val == nil {
result[key] = s
}
}
if len(result) == 0 && len(v) > 0 {
return nil
}
return result
case []any:
result := make([]any, len(v))
for i, elem := range v {
result[i] = stripFields(structpath.NewIndex(path, i), elem, drop)
}
return result
}
return value
}
38 changes: 2 additions & 36 deletions bundle/direct/bundle_plan.go
Original file line number Diff line number Diff line change
Expand Up @@ -1189,7 +1189,7 @@ func shouldSkipBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *str
if cfg == nil || ch.Old != nil || ch.New != nil || ch.Remote == nil {
return "", false
}
if matchesAnyBackendDefault(cfg, path, ch.Remote) {
if cfg.IsBackendDefault(path, ch.Remote) {
return deployplan.ReasonBackendDefault, true
}

Expand All @@ -1205,47 +1205,13 @@ func shouldSkipBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *str
iter := rv.MapRange()
for iter.Next() {
childPath := structpath.NewBracketString(path, iter.Key().String())
if !matchesAnyBackendDefault(cfg, childPath, iter.Value().Interface()) {
if !cfg.IsBackendDefault(childPath, iter.Value().Interface()) {
return "", false
}
}
return deployplan.ReasonBackendDefault, true
}

// matchesAnyBackendDefault reports whether the remote value at path matches any of
// the resource's configured backend-default rules (and the rule's allowed values,
// if specified).
func matchesAnyBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, remote any) bool {
for _, rule := range cfg.BackendDefaults {
if !path.HasPatternPrefix(rule.Field) {
continue
}
if len(rule.Values) == 0 {
return true
}
if matchesAllowedValue(remote, rule.Values) {
return true
}
}
return false
}

// matchesAllowedValue checks if the remote value matches one of the allowed JSON values.
// Each json.RawMessage is unmarshaled into the same type as remote for comparison.
func matchesAllowedValue(remote any, values []json.RawMessage) bool {
remoteType := reflect.TypeOf(remote)
for _, raw := range values {
candidate := reflect.New(remoteType).Interface()
if err := json.Unmarshal(raw, candidate); err != nil {
continue
}
if structdiff.IsEqual(remote, reflect.ValueOf(candidate).Elem().Interface()) {
return true
}
}
return false
}

func allEmpty(values ...any) bool {
for _, v := range values {
if v == nil {
Expand Down
35 changes: 35 additions & 0 deletions bundle/direct/dresources/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@ import (
"fmt"
"io/fs"
"path"
"reflect"
"strings"
"sync"

"github.com/databricks/cli/libs/structs/structdiff"
"github.com/databricks/cli/libs/structs/structpath"
"go.yaml.in/yaml/v3"
)
Expand Down Expand Up @@ -49,6 +51,39 @@ func (b *BackendDefaultRule) UnmarshalYAML(unmarshal func(any) error) error {
return nil
}

// IsBackendDefault reports whether remote, the remote value at path, matches any of
// the backend-default rules (and the rule's allowed values, if specified).
func (c *ResourceLifecycleConfig) IsBackendDefault(path *structpath.PathNode, remote any) bool {
for _, rule := range c.BackendDefaults {
if !path.HasPatternPrefix(rule.Field) {
continue
}
if len(rule.Values) == 0 {
return true
}
if matchesAllowedValue(remote, rule.Values) {
return true
}
}
return false
}

// matchesAllowedValue checks if the remote value matches one of the allowed JSON values.
// Each json.RawMessage is unmarshaled into the same type as remote for comparison.
func matchesAllowedValue(remote any, values []json.RawMessage) bool {
remoteType := reflect.TypeOf(remote)
for _, raw := range values {
candidate := reflect.New(remoteType).Interface()
if err := json.Unmarshal(raw, candidate); err != nil {
continue
}
if structdiff.IsEqual(remote, reflect.ValueOf(candidate).Elem().Interface()) {
return true
}
}
return false
}

// RemoteAdditionRule marks a sub-object whose contents the backend co-owns whenever the
// object's WhenSet field is set. Inside such an object, a field the config never declared
// coming back set from the remote is an addition by the backend, not drift.
Expand Down
Loading