diff --git a/.nextchanges/notable-changes/4872.md b/.nextchanges/notable-changes/4872.md new file mode 100644 index 00000000000..5798612d9e8 --- /dev/null +++ b/.nextchanges/notable-changes/4872.md @@ -0,0 +1 @@ +* `lifecycle.prevent_destroy` now also protects a resource after it is removed from the configuration: `bundle deploy` records the flag in the deployment state, and `bundle deploy` and `bundle destroy` refuse to delete a resource whose last deployment had it set. To delete such a resource, add it back with `lifecycle.prevent_destroy: false`, deploy, then remove it again. ([#7499](https://github.com/databricks/cli/pull/7499)) diff --git a/acceptance/bundle/dms/prevent-destroy/databricks.yml.tmpl b/acceptance/bundle/dms/prevent-destroy/databricks.yml.tmpl new file mode 100644 index 00000000000..d97a2fb63b9 --- /dev/null +++ b/acceptance/bundle/dms/prevent-destroy/databricks.yml.tmpl @@ -0,0 +1,7 @@ +bundle: + name: dms-prevent-destroy-$UNIQUE_NAME +experimental: + deployment_history: true + +include: + - resources/*.yml diff --git a/acceptance/bundle/dms/prevent-destroy/out.test.toml b/acceptance/bundle/dms/prevent-destroy/out.test.toml new file mode 100644 index 00000000000..84b22ba6735 --- /dev/null +++ b/acceptance/bundle/dms/prevent-destroy/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/prevent-destroy/output.txt b/acceptance/bundle/dms/prevent-destroy/output.txt new file mode 100644 index 00000000000..c2c7278a757 --- /dev/null +++ b/acceptance/bundle/dms/prevent-destroy/output.txt @@ -0,0 +1,89 @@ + +=== Deploy a protected job: prevent_destroy is recorded in the state the service holds + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-prevent-destroy-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //versions/1/operations +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[FOO_ID]", + "sequence_id": 0, + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-prevent-destroy-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}},\"prevent_destroy\":true}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} + +=== Removing the job from the config: deploy and destroy refuse to delete it + +>>> [CLI] bundle plan +delete jobs.foo + +Plan: 0 to add, 0 to change, 1 to delete, 0 unchanged + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-prevent-destroy-[UNIQUE_NAME]/default/files... +Error: resources.jobs.foo was removed from the configuration, but its last deployment has lifecycle.prevent_destroy set. To delete it, add it back with lifecycle.prevent_destroy: false and deploy, then remove it again + +Files: 1 uploaded, 1 deleted + +>>> musterr [CLI] bundle destroy --auto-approve +Error: resources.jobs.foo was removed from the configuration, but its last deployment has lifecycle.prevent_destroy set. To delete it, add it back with lifecycle.prevent_destroy: false and deploy, then remove it again + + +=== Adding it back with prevent_destroy: false leaves the job unchanged, but records the flag as an update of its state + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-prevent-destroy-[UNIQUE_NAME]/default/files... +Files: 1 uploaded, 1 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> print_requests.py --dms //versions/2/operations +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/jobs.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[FOO_ID]", + "sequence_id": 0, + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[DEPLOYMENT_ID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-prevent-destroy-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} + +=== The job can now be removed + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-prevent-destroy-[UNIQUE_NAME]/default/files... +Deleted jobs.foo +Files: 0 uploaded, 1 deleted +Resources: 0 created, 0 changed, 1 deleted, 0 unchanged + +>>> print_requests.py --dms //versions/3/operations +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/operations/jobs.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": 0, + "status": "OPERATION_STATUS_SUCCEEDED" + } +} diff --git a/acceptance/bundle/dms/prevent-destroy/resources/job.yml b/acceptance/bundle/dms/prevent-destroy/resources/job.yml new file mode 100644 index 00000000000..872e5647af3 --- /dev/null +++ b/acceptance/bundle/dms/prevent-destroy/resources/job.yml @@ -0,0 +1,6 @@ +resources: + jobs: + foo: + name: foo + lifecycle: + prevent_destroy: true diff --git a/acceptance/bundle/dms/prevent-destroy/script b/acceptance/bundle/dms/prevent-destroy/script new file mode 100644 index 00000000000..4707ca4fb9e --- /dev/null +++ b/acceptance/bundle/dms/prevent-destroy/script @@ -0,0 +1,26 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy a protected job: prevent_destroy is recorded in the state the service holds" +trace $CLI bundle deploy +replace_ids.py +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-prevent-destroy-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +add_repl "$deployment_id" DEPLOYMENT_ID +trace print_requests.py --dms //versions/1/operations + +title "Removing the job from the config: deploy and destroy refuse to delete it" +mv resources removed +trace $CLI bundle plan +trace musterr $CLI bundle deploy +trace musterr $CLI bundle destroy --auto-approve + +title "Adding it back with prevent_destroy: false leaves the job unchanged, but records the flag as an update of its state" +mv removed resources +update_file.py resources/job.yml 'prevent_destroy: true' 'prevent_destroy: false' +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py --dms //versions/2/operations + +title "The job can now be removed" +rm resources/job.yml +trace $CLI bundle deploy +trace print_requests.py --dms //versions/3/operations diff --git a/acceptance/bundle/lifecycle/prevent-destroy/out.txt b/acceptance/bundle/lifecycle/prevent-destroy/out.txt index a7082af177b..a897f9d3d7d 100644 --- a/acceptance/bundle/lifecycle/prevent-destroy/out.txt +++ b/acceptance/bundle/lifecycle/prevent-destroy/out.txt @@ -46,6 +46,48 @@ Recreated schemas.my_schema Files: 3 uploaded, 0 deleted Resources: 2 created, 0 changed, 2 deleted, 0 unchanged +>>> [CLI] bundle plan -o json +{ + "resources.pipelines.my_pipelines": { + "action": "skip", + "prevent_destroy": true, + "prevent_destroy_changed": true + }, + "resources.schemas.my_schema": { + "action": "skip", + "prevent_destroy": true, + "prevent_destroy_changed": true + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/prevent-destroy/default/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> errcode [CLI] bundle plan +delete pipelines.my_pipelines +delete schemas.my_schema + +Plan: 0 to add, 0 to change, 2 to delete, 0 unchanged + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/prevent-destroy/default/files... +Error: resources.pipelines.my_pipelines was removed from the configuration, but its last deployment has lifecycle.prevent_destroy set. To delete it, add it back with lifecycle.prevent_destroy: false and deploy, then remove it again +resources.schemas.my_schema was removed from the configuration, but its last deployment has lifecycle.prevent_destroy set. To delete it, add it back with lifecycle.prevent_destroy: false and deploy, then remove it again + +Files: 3 uploaded, 2 deleted + +>>> musterr [CLI] bundle destroy --auto-approve +Error: resources.pipelines.my_pipelines was removed from the configuration, but its last deployment has lifecycle.prevent_destroy set. To delete it, add it back with lifecycle.prevent_destroy: false and deploy, then remove it again +resources.schemas.my_schema was removed from the configuration, but its last deployment has lifecycle.prevent_destroy set. To delete it, add it back with lifecycle.prevent_destroy: false and deploy, then remove it again + + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/prevent-destroy/default/files... +Files: 3 uploaded, 2 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + >>> errcode [CLI] bundle plan delete pipelines.my_pipelines delete schemas.my_schema diff --git a/acceptance/bundle/lifecycle/prevent-destroy/script b/acceptance/bundle/lifecycle/prevent-destroy/script index 94c074cfc0b..fa0741111d2 100644 --- a/acceptance/bundle/lifecycle/prevent-destroy/script +++ b/acceptance/bundle/lifecycle/prevent-destroy/script @@ -22,11 +22,25 @@ update_file.py resources/pipeline.yml 'prevent_destroy: true' 'prevent_destroy: update_file.py resources/schema.yml 'prevent_destroy: true' 'prevent_destroy: false' trace errcode $CLI bundle plan >>out.txt 2>&1 trace $CLI bundle deploy --auto-approve >>out.txt 2>&1 + +# Setting prevent_destroy back to true leaves the resources unchanged, but the plan marks them +# so that the deploy records the flag in the state update_file.py resources/pipeline.yml 'prevent_destroy: false' 'prevent_destroy: true' update_file.py resources/schema.yml 'prevent_destroy: false' 'prevent_destroy: true' +{ trace $CLI bundle plan -o json | jq '.plan | map_values({action, prevent_destroy, prevent_destroy_changed})'; } >>out.txt 2>&1 +trace $CLI bundle deploy >>out.txt 2>&1 +# Removing the pipeline and schema: their last deployment had prevent_destroy set, so neither deploy nor destroy may delete them +mv resources removed +trace errcode $CLI bundle plan >>out.txt 2>&1 +trace musterr $CLI bundle deploy --auto-approve >>out.txt 2>&1 +trace musterr $CLI bundle destroy --auto-approve >>out.txt 2>&1 -# Removing the pipeline and schema, deploy must succeed +# Adding them back with prevent_destroy: false and deploying records that; only then can they be removed +mv removed resources +update_file.py resources/pipeline.yml 'prevent_destroy: true' 'prevent_destroy: false' +update_file.py resources/schema.yml 'prevent_destroy: true' 'prevent_destroy: false' +trace $CLI bundle deploy >>out.txt 2>&1 rm resources/pipeline.yml resources/schema.yml trace errcode $CLI bundle plan >>out.txt 2>&1 trace $CLI bundle deploy --auto-approve >>out.txt 2>&1 diff --git a/bundle/deployplan/action.go b/bundle/deployplan/action.go index 408e2a143f5..0027c623149 100644 --- a/bundle/deployplan/action.go +++ b/bundle/deployplan/action.go @@ -15,6 +15,13 @@ type Action struct { // StateOnly mirrors PlanEntry.StateOnly: applying the action only changes the state // (a delete of a resource without DoDelete, or a re-key). StateOnly bool + // PreventDestroy mirrors PlanEntry.PreventDestroy: lifecycle.prevent_destroy as + // configured or, for a resource no longer in the configuration, as recorded by + // its last deployment. + PreventDestroy bool + // PreventDestroyChanged mirrors PlanEntry.PreventDestroyChanged: the skip still + // rewrites the state entry with the new PreventDestroy. + PreventDestroyChanged bool } func (a Action) String() string { diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 44d6e9f0f12..10688b3c6d0 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -146,10 +146,19 @@ type PlanEntry struct { // - an Update of an otherwise unchanged resource whose state lacks its internal ID (recorded // before the ID was, or by bind): applying it records the ID from the remote state. It is // likewise left out of the textual plan and counted as unchanged. - StateOnly bool `json:"state_only,omitempty"` - NewState *structvar.StructVarJSON `json:"new_state,omitempty"` - RemoteState any `json:"remote_state,omitempty"` - Changes Changes `json:"changes,omitempty"` + StateOnly bool `json:"state_only,omitempty"` + // PreventDestroy is the resource's lifecycle.prevent_destroy: as configured, or for a Delete + // entry of a resource the configuration no longer declares, as its last deployment recorded + // it in the state. Applying the entry persists it with the resource state so that a later + // plan can still read it once the resource is removed from the configuration. + PreventDestroy bool `json:"prevent_destroy,omitempty"` + // PreventDestroyChanged is set on Skip entries whose recorded PreventDestroy differs from + // the configured one. The resource itself is unchanged, so applying the entry only rewrites + // its state entry; no backend call is made. + PreventDestroyChanged bool `json:"prevent_destroy_changed,omitempty"` + NewState *structvar.StructVarJSON `json:"new_state,omitempty"` + RemoteState any `json:"remote_state,omitempty"` + Changes Changes `json:"changes,omitempty"` } // IsStateOnly reports whether applying this entry makes no backend call and only changes the @@ -267,10 +276,12 @@ func (p *Plan) GetActions() []Action { actions := make([]Action, 0, len(p.Plan)) for key, entry := range p.Plan { actions = append(actions, Action{ - ResourceKey: key, - ActionType: entry.Action, - Gone: entry.Gone, - StateOnly: entry.StateOnly, + ResourceKey: key, + ActionType: entry.Action, + Gone: entry.Gone, + StateOnly: entry.StateOnly, + PreventDestroy: entry.PreventDestroy, + PreventDestroyChanged: entry.PreventDestroyChanged, }) } diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 5b093a61d4c..121e532798c 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -99,7 +99,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn, nil) + err = d.saveState(ctx, db, newID, newState, nil) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -197,7 +197,7 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("deleting state id=%s: %w", id, err) } } else { - err = d.saveState(ctx, db, id, newState, d.DependsOn, planEntry.RemoteState) + err = d.saveState(ctx, db, id, newState, planEntry.RemoteState) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -246,7 +246,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn, planEntry.RemoteState) + err = d.saveState(ctx, db, newID, newState, planEntry.RemoteState) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -276,7 +276,7 @@ func (d *DeploymentUnit) Rekey(ctx context.Context, db *dstate.DeploymentState, } log.Infof(ctx, "Re-keyed %s id=%#v (previously %#v)", d.ResourceKey, newID, oldID) - err = d.saveState(ctx, db, newID, newState, d.DependsOn, planEntry.RemoteState) + err = d.saveState(ctx, db, newID, newState, planEntry.RemoteState) if err != nil { return fmt.Errorf("saving state id=%s: %w", newID, err) } @@ -286,7 +286,7 @@ func (d *DeploymentUnit) Rekey(ctx context.Context, db *dstate.DeploymentState, // RecordState saves the state at its ID without a backend call. A state-only update does this to // record the internal ID the state lacks, from the plan's remote state. func (d *DeploymentUnit) RecordState(ctx context.Context, db *dstate.DeploymentState, id string, newState any, planEntry *deployplan.PlanEntry) error { - err := d.saveState(ctx, db, id, newState, d.DependsOn, planEntry.RemoteState) + err := d.saveState(ctx, db, id, newState, planEntry.RemoteState) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -365,7 +365,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = d.saveState(ctx, db, id, newState, d.DependsOn, entry.RemoteState) + err = d.saveState(ctx, db, id, newState, entry.RemoteState) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -373,6 +373,18 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return nil } +// RecordPreventDestroy rewrites the resource's state entry with the planned lifecycle.prevent_destroy, +// keeping its id and state. For a resource the plan skips: nothing is sent to the backend, but the +// flag changed and the next plan must read the new value from the state. +func (d *DeploymentUnit) RecordPreventDestroy(ctx context.Context, db *dstate.DeploymentState) error { + entry, ok := db.GetResourceEntry(d.ResourceKey) + if !ok { + return errors.New("state entry not found") + } + log.Infof(ctx, "Recording lifecycle.prevent_destroy=%t for %s id=%#v", d.PreventDestroy, d.ResourceKey, entry.ID) + return db.SaveState(ctx, d.ResourceKey, entry.ID, entry.State, d.DependsOn, d.PreventDestroy) +} + // saveState compacts the state (replacing fields declared in hashed_fields // with content hashes, see dresources.CompactState) before persisting it. Fields already // smaller than a hash placeholder are persisted as is. Sensitive fields are replaced by a @@ -381,7 +393,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, // It first records the internal ID in state (see ResourceLifecycleConfig.InternalIDField), from the // remote state the operation returned or else from planRemote, the object as the plan read it (nil // for a new object). -func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry, planRemote any) error { +func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state, planRemote any) error { remote := d.RemoteState if remote == nil { remote = planRemote @@ -397,7 +409,7 @@ func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentSta if err := zeroSensitiveFields(d.Adapter, compacted); err != nil { return fmt.Errorf("redacting state: %w", err) } - return db.SaveState(ctx, d.ResourceKey, newID, compacted, dependsOn) + return db.SaveState(ctx, d.ResourceKey, newID, compacted, d.DependsOn, d.PreventDestroy) } func parseState(destType reflect.Type, raw json.RawMessage) (any, error) { diff --git a/bundle/direct/apply_test.go b/bundle/direct/apply_test.go index 24304292ddc..fc877aa839b 100644 --- a/bundle/direct/apply_test.go +++ b/bundle/direct/apply_test.go @@ -72,13 +72,41 @@ func TestCreateFailsOnEmptyID(t *testing.T) { assert.False(t, ok) } +func TestRecordPreventDestroy(t *testing.T) { + adapter, err := dresources.NewAdapter((*emptyIDResource)(nil), "fake", nil) + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "resources.json") + + // A deploy records the resource without the flag. + db := &dstate.DeploymentState{} + require.NoError(t, db.Open(t.Context(), path, dstate.WritingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{})) + require.NoError(t, db.SaveState(t.Context(), "resources.fakes.foo", "id", &emptyIDState{Name: "foo"}, nil, false)) + finalizeEmptyIDState(t, db) + + // The next deploy skips the unchanged resource but plans prevent_destroy: only the flag moves. + db = &dstate.DeploymentState{} + require.NoError(t, db.Open(t.Context(), path, dstate.WritingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{})) + d := &DeploymentUnit{ResourceKey: "resources.fakes.foo", Adapter: adapter, PreventDestroy: true} + require.NoError(t, d.RecordPreventDestroy(t.Context(), db)) + finalizeEmptyIDState(t, db) + + db = &dstate.DeploymentState{} + require.NoError(t, db.Open(t.Context(), path, dstate.ReadingFile, dstate.WithRecovery(false), dstate.OpenDmsArgs{})) + defer finalizeEmptyIDState(t, db) + entry, ok := db.GetResourceEntry("resources.fakes.foo") + require.True(t, ok) + assert.Equal(t, "id", entry.ID) + assert.JSONEq(t, `{"name":"foo"}`, string(entry.State)) + assert.True(t, entry.PreventDestroy) +} + func TestUpdateWithIDFailsOnEmptyID(t *testing.T) { adapter, err := dresources.NewAdapter((*emptyIDResource)(nil), "fake", nil) require.NoError(t, err) db := newEmptyIDDeploymentState(t) defer finalizeEmptyIDState(t, db) d := &DeploymentUnit{ResourceKey: "resources.fakes.foo", Adapter: adapter} - require.NoError(t, db.SaveState(t.Context(), d.ResourceKey, "old-id", &emptyIDState{Name: "foo"}, nil)) + require.NoError(t, db.SaveState(t.Context(), d.ResourceKey, "old-id", &emptyIDState{Name: "foo"}, nil, false)) err = d.UpdateWithID(t.Context(), db, "old-id", &emptyIDState{Name: "bar"}, nil) require.EqualError(t, err, "update returned an empty resource id") diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 73852cdde00..6b3c96d4c41 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -103,7 +103,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Save state with ID and empty state (like migrate does) - err = b.StateDB.SaveState(ctx, resourceKey, resourceID, struct{}{}, nil) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, struct{}{}, nil, false) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -138,8 +138,10 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac sv, ok := b.StateCache.Load(resourceKey) if ok && sv != nil { var dependsOn []deployplan.DependsOnEntry + var preventDestroy bool if entry != nil { dependsOn = entry.DependsOn + preventDestroy = entry.PreventDestroy } // Copy etag from remote state for resources that use etag-based drift @@ -178,7 +180,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac return nil, err } - err = b.StateDB.SaveState(ctx, resourceKey, resourceID, compacted, dependsOn) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, compacted, dependsOn, preventDestroy) if err != nil { os.Remove(tmpStatePath) return nil, err diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 60e1b66bb47..d9881f7c665 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -108,10 +108,11 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } d := &DeploymentUnit{ - ResourceKey: resourceKey, - Adapter: adapter, - DependsOn: entry.DependsOn, - MaxWait: unitWait, + ResourceKey: resourceKey, + Adapter: adapter, + DependsOn: entry.DependsOn, + PreventDestroy: entry.PreventDestroy, + MaxWait: unitWait, } if action == deployplan.Delete { @@ -138,6 +139,15 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // We don't keep NewState around for 'skip' nodes + if action == deployplan.Skip && entry.PreventDestroyChanged { + // The resource is left alone, but its state entry must pick up the new + // lifecycle.prevent_destroy for a later plan to read. + if err := d.RecordPreventDestroy(ctx, &b.StateDB); err != nil { + logdiag.LogError(ctx, fmt.Errorf("cannot record lifecycle.prevent_destroy for %s: %w", resourceKey, err)) + return false + } + } + if action != deployplan.Skip { if !b.resolveReferences(ctx, resourceKey, entry, errorPrefix, false) { return false diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 43a58291ab2..93678575835 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -437,6 +437,12 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } + // lifecycle.prevent_destroy is kept next to the state, not in it, so toggling it alone + // leaves the resource unchanged. Mark the skip so the apply still rewrites the state entry. + if action == deployplan.Skip && dbentry.PreventDestroy != entry.PreventDestroy { + entry.PreventDestroyChanged = true + } + entry.Action = action return true }) @@ -1764,8 +1770,9 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root } e := deployplan.PlanEntry{ - DependsOn: dependsOn, - NewState: newStateJSON, + DependsOn: dependsOn, + PreventDestroy: LifecyclePreventDestroy(inputConfig), + NewState: newStateJSON, } p.Plan[node] = &e @@ -1785,8 +1792,9 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root } p.Plan[n] = &deployplan.PlanEntry{ - Action: deployplan.Delete, - DependsOn: entry.DependsOn, + Action: deployplan.Delete, + DependsOn: entry.DependsOn, + PreventDestroy: entry.PreventDestroy, } } @@ -1881,6 +1889,16 @@ func (b *DeploymentBundle) getAdapterForKey(resourceKey string) (*dresources.Ada const lifecycleDependsOnLabel = "depends_on" +// LifecyclePreventDestroy returns lifecycle.prevent_destroy of a resource, given its config as +// returned by [config.Root.GetResourceConfig]. Nodes without a lifecycle (permissions, grants) +// are never protected on their own: they are deleted with their parent, which is. +func LifecyclePreventDestroy(inputConfig any) bool { + resource, ok := inputConfig.(interface { + GetLifecycle() resources.LifecycleConfig + }) + return ok && resource.GetLifecycle().HasPreventDestroy() +} + // LifecycleDependsOn returns the edges declared in lifecycle.depends_on of a resource, // given its config as returned by [config.Root.GetResourceConfig]. func LifecycleDependsOn(configRoot *config.Root, inputConfig any) ([]deployplan.DependsOnEntry, error) { diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 57e10b8d4b4..b123265ed44 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -9,11 +9,13 @@ import ( ) // RecordedState is what the CLI serializes into the DMS operation's state field. It wraps the -// config so depends_on survives the round trip: DMS has no field for dependency edges, and -// nesting them in the config would collide with resource fields of the same name. +// config so depends_on and prevent_destroy survive the round trip: DMS has no field for dependency +// edges or lifecycle settings, and nesting them in the config would collide with resource fields +// of the same name. type RecordedState struct { - State json.RawMessage `json:"state"` - DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` + State json.RawMessage `json:"state"` + DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` + PreventDestroy bool `json:"prevent_destroy,omitempty"` } // applyDMSState fills in the resource state from what DMS recorded. For a deployment that uses DMS @@ -54,8 +56,9 @@ func stateEntry(res dms.Resource) (ResourceEntry, error) { } return ResourceEntry{ - ID: res.ID, - State: recorded.State, - DependsOn: recorded.DependsOn, + ID: res.ID, + State: recorded.State, + DependsOn: recorded.DependsOn, + PreventDestroy: recorded.PreventDestroy, }, nil } diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index ab4e1bc017f..4668b038c6d 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -13,7 +13,7 @@ import ( func TestApplyDMSState(t *testing.T) { // The service stores state as an opaque string, so the envelope the write path sent // arrives verbatim. - envelope := `{"state":{"name":"foo"},"depends_on":[{"node":"resources.pipelines.bar","label":"${resources.pipelines.bar.id}"}]}` + envelope := `{"state":{"name":"foo"},"depends_on":[{"node":"resources.pipelines.bar","label":"${resources.pipelines.bar.id}"}],"prevent_destroy":true}` // What the file loaded, for the cases that check it is replaced or survives. fileState := map[string]ResourceEntry{ @@ -28,18 +28,20 @@ func TestApplyDMSState(t *testing.T) { wantErr string }{ { - name: "the envelope is unwrapped, depends_on and all", + name: "the envelope is unwrapped, depends_on and prevent_destroy and all", recorded: []dms.Resource{ {Key: "resources.jobs.foo", ID: "123", State: envelope}, {Key: "resources.pipelines.bar", ID: "456"}, }, - // depends_on comes back from the envelope, so a bundle whose local state was - // wiped still has the edges needed for delete ordering. + // depends_on and prevent_destroy come back from the envelope, so a bundle whose local + // state was wiped still has the edges needed for delete ordering and the flag that + // protects a resource removed from the config. want: map[string]ResourceEntry{ "resources.jobs.foo": { - ID: "123", - State: json.RawMessage(`{"name":"foo"}`), - DependsOn: []deployplan.DependsOnEntry{{Node: "resources.pipelines.bar", Label: "${resources.pipelines.bar.id}"}}, + ID: "123", + State: json.RawMessage(`{"name":"foo"}`), + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.pipelines.bar", Label: "${resources.pipelines.bar.id}"}}, + PreventDestroy: true, }, "resources.pipelines.bar": {ID: "456"}, }, diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index de00e76233a..9718be81009 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -169,6 +169,11 @@ type ResourceEntry struct { ID string `json:"__id__"` State json.RawMessage `json:"state"` DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` + + // PreventDestroy is lifecycle.prevent_destroy as of the deploy that wrote the entry. It is + // kept with the state because once the resource is removed from the configuration there is + // nowhere else to read it from, and that is exactly when it must still protect the resource. + PreventDestroy bool `json:"prevent_destroy,omitempty"` } type WALEntry struct { @@ -289,7 +294,7 @@ func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause e var recorded json.RawMessage if entry, ok := db.GetResourceEntry(resourceKey); resourceID != "" && ok && len(entry.State) > 0 { var err error - recorded, err = json.Marshal(RecordedState{State: entry.State, DependsOn: entry.DependsOn}) + recorded, err = json.Marshal(RecordedState{State: entry.State, DependsOn: entry.DependsOn, PreventDestroy: entry.PreventDestroy}) if err != nil { // Nothing the caller can act on, so record the failure without the state. recorded = nil @@ -314,7 +319,7 @@ func NewDatabase(lineage string, serial int) Database { } // SaveState records the resource's state after an operation was applied to it. -func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { +func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry, preventDestroy bool) error { db.AssertOpenedForWrite() db.mu.Lock() defer db.mu.Unlock() @@ -328,9 +333,10 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta return err } entry := ResourceEntry{ - ID: newID, - State: json.RawMessage(jsonMessage), - DependsOn: dependsOn, + ID: newID, + State: json.RawMessage(jsonMessage), + DependsOn: dependsOn, + PreventDestroy: preventDestroy, } switch db.mode { @@ -344,7 +350,7 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta db.Data.State[key] = entry db.stateIDs[key] = newID if buf := db.operationBuffer; buf != nil { - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, PreventDestroy: preventDestroy}) if err != nil { return err } @@ -359,7 +365,7 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta } db.stateIDs[key] = newID if buf := db.operationBuffer; buf != nil { - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, PreventDestroy: preventDestroy}) if err != nil { return err } diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index c5bfea2783b..fdb14373f9c 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -27,7 +27,7 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WritingFile, WithRecovery(true), OpenDmsArgs{})) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "val"}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "val"}, nil, false)) mustFinalize(t, &db) // Re-open and verify persisted data. @@ -310,7 +310,7 @@ func TestCLIVersionRecordsLastWriter(t *testing.T) { var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WritingFile, WithRecovery(true), OpenDmsArgs{})) - require.NoError(t, db.SaveState(t.Context(), "resources.jobs.my_job", "123", map[string]string{"k": "v"}, nil)) + require.NoError(t, db.SaveState(t.Context(), "resources.jobs.my_job", "123", map[string]string{"k": "v"}, nil, false)) mustFinalize(t, &db) var reopened DeploymentState @@ -349,7 +349,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WritingFile, WithRecovery(true), OpenDmsArgs{})) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, false)) mustFinalize(t, &db) var committed DeploymentState @@ -399,7 +399,7 @@ func TestDeleteState(t *testing.T) { var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WritingFile, WithRecovery(true), OpenDmsArgs{})) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, false)) mustFinalize(t, &db) var db2 DeploymentState @@ -432,7 +432,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Upgrading to write reuses the same lineage (it goes into the WAL header), // and a write makes it durable. require.NoError(t, db.UpgradeToWrite()) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, false)) mustFinalize(t, &db) // Re-open: the persisted lineage matches the one read before the write. @@ -462,6 +462,30 @@ func TestOpenFailureLeavesStateClosed(t *testing.T) { mustFinalize(t, &db) } +func TestPreventDestroyRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + // A state written before the flag was recorded loads as unprotected. + seed := `{"state_version":3,"lineage":"test-lineage","serial":1,"state":{"resources.schemas.legacy":{"__id__":"main.legacy","state":{}}}}` + require.NoError(t, os.WriteFile(path, []byte(seed), 0o600)) + + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WritingFile, WithRecovery(true), OpenDmsArgs{})) + legacy, ok := db.GetResourceEntry("resources.schemas.legacy") + require.True(t, ok) + assert.False(t, legacy.PreventDestroy) + + require.NoError(t, db.SaveState(t.Context(), "resources.schemas.protected", "main.protected", map[string]string{}, nil, true)) + mustFinalize(t, &db) + + var reopened DeploymentState + require.NoError(t, reopened.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) + protected, ok := reopened.GetResourceEntry("resources.schemas.protected") + require.True(t, ok) + assert.True(t, protected.PreventDestroy) + mustFinalize(t, &reopened) +} + func TestGetSerialWithHistoryUsesLatestCounter(t *testing.T) { for _, mode := range []Mode{ReadingFile, WritingDeploymentHistoryShadow} { db := DeploymentState{mode: mode, Data: NewDatabase("", 4), VersionID: 7} diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index 84990d0883e..cdb352f0385 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -37,6 +37,9 @@ type DeploymentUnit struct { // DependsOn lists resources this resource depends on (persisted in state). DependsOn []deployplan.DependsOnEntry + // PreventDestroy is the resource's lifecycle.prevent_destroy (persisted in state). + PreventDestroy bool + // MaxWait caps how long to wait for this resource to reach its target state. Must be set // to maxWaitUnset to fall back to the resource's own timeout: the zero value is a valid // cap meaning "do not wait at all". See resourceMaxWait and unitMaxWait. diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 7c625acf1d9..250ea0dc6a5 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -37,7 +37,7 @@ func SeedState(ctx context.Context, stateDB *dstate.DeploymentState, tfState *TF return nil } for key, id := range tfState.IDs { - if err := stateDB.SaveState(ctx, key, id, json.RawMessage("{}"), nil); err != nil { + if err := stateDB.SaveState(ctx, key, id, json.RawMessage("{}"), nil, false); err != nil { return fmt.Errorf("seeding migrated state for %s: %w", key, err) } } @@ -287,7 +287,7 @@ func BuildStateFromTF( return warningsSeen, fmt.Errorf("%s: compacting state: %w", node, err) } - if err := stateDB.SaveState(ctx, node, id, compacted, dependsOn); err != nil { + if err := stateDB.SaveState(ctx, node, id, compacted, dependsOn, direct.LifecyclePreventDestroy(inputConfig)); err != nil { return warningsSeen, fmt.Errorf("%s: SaveState: %w", node, err) } } diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 38210913ce8..4abdbec9db1 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -26,15 +26,23 @@ import ( // stagedOperations lists the resources the plan will touch, for CreateVersion to stage an // operation each. Skipped and undefined actions are left out: nothing is applied for them, so -// their operations would stay pending and the service would hold no state for them. +// their operations would stay pending and the service would hold no state for them. The +// exception is a skip that rewrites the state entry for a changed lifecycle.prevent_destroy: +// the service's copy of the state must follow, so it is recorded as an update. func stagedOperations(plan *deployplan.Plan) ([]bundledeployments.StagedOperation, error) { actions := plan.GetActions() staged := make([]bundledeployments.StagedOperation, 0, len(actions)) for _, action := range actions { - if action.ActionType == deployplan.Skip || action.ActionType == deployplan.Undefined { + planned := action.ActionType + if planned == deployplan.Skip && action.PreventDestroyChanged { + // The resource is left alone but its recorded state changes, and the service has no + // action type for that: an update is the closest description. + planned = deployplan.Update + } + if planned == deployplan.Skip || planned == deployplan.Undefined { continue } - actionType, err := actionToSDK(action.ActionType) + actionType, err := actionToSDK(planned) if err != nil { return nil, fmt.Errorf("%s: %w", action.ResourceKey, err) } diff --git a/bundle/phases/dms_test.go b/bundle/phases/dms_test.go index 6b045f863ae..e5ec23fefb8 100644 --- a/bundle/phases/dms_test.go +++ b/bundle/phases/dms_test.go @@ -32,17 +32,21 @@ func TestStagedOperationsCoversEveryTouchedResource(t *testing.T) { func TestStagedOperationsLeavesOutUntouchedResources(t *testing.T) { // A skipped resource is never applied, so staging it would leave an operation pending for - // the life of the version. Undefined is not a real action either. + // the life of the version. Undefined is not a real action either. The one skip that is + // applied, as a rewrite of its state entry for a changed lifecycle.prevent_destroy, is + // staged as an update so the service's copy of the state follows. plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ - "resources.jobs.touched": {Action: deployplan.Update}, - "resources.jobs.unchanged": {Action: deployplan.Skip}, - "resources.jobs.unknown": {Action: deployplan.Undefined}, + "resources.jobs.touched": {Action: deployplan.Update}, + "resources.jobs.unchanged": {Action: deployplan.Skip}, + "resources.jobs.reprotected": {Action: deployplan.Skip, PreventDestroyChanged: true}, + "resources.jobs.unknown": {Action: deployplan.Undefined}, }} staged, err := stagedOperations(plan) require.NoError(t, err) assert.Equal(t, []bundledeployments.StagedOperation{ + {ResourceKey: "jobs.reprotected", ActionType: bundledeployments.OperationActionTypeOperationActionTypeUpdate}, {ResourceKey: "jobs.touched", ActionType: bundledeployments.OperationActionTypeOperationActionTypeUpdate}, }, staged) } diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 4be81a102d2..1284b1877f3 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -51,6 +51,10 @@ func pipelineDeletionCascades(b *bundle.Bundle, action deployplan.Action) (bool, // checkForPreventDestroy checks if the resource has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. // If it does, it returns an error. +// +// For a resource still in the configuration the configured value decides. A resource that was +// removed from the configuration is protected by the value its last deployment recorded in the +// state, which the plan carries; the configuration cannot be consulted for it anymore. func checkForPreventDestroy(b *bundle.Bundle, actions []deployplan.Action) error { root := b.Config.Value() var errs []error @@ -64,6 +68,13 @@ func checkForPreventDestroy(b *bundle.Bundle, actions []deployplan.Action) error return fmt.Errorf("failed to parse %q", action.ResourceKey) } + if _, err := dyn.GetByPath(root, path); err != nil { + if action.PreventDestroy { + errs = append(errs, fmt.Errorf("%s was removed from the configuration, but its last deployment has lifecycle.prevent_destroy set. To delete it, add it back with lifecycle.prevent_destroy: false and deploy, then remove it again", action.ResourceKey)) + } + continue + } + path = append(path, dyn.Key("lifecycle"), dyn.Key("prevent_destroy")) preventDestroyV, err := dyn.GetByPath(root, path) diff --git a/bundle/phases/plan_test.go b/bundle/phases/plan_test.go index 772f7401f9b..a0d86988a85 100644 --- a/bundle/phases/plan_test.go +++ b/bundle/phases/plan_test.go @@ -187,6 +187,53 @@ func TestCheckPreventDestroyNoError(t *testing.T) { require.NoError(t, err) } +func TestCheckPreventDestroyRemovedFromConfig(t *testing.T) { + // resources.schemas.kept is still declared, without the flag. resources.schemas.removed is + // only in the state, so the plan carries the flag its last deployment recorded. + b := &bundle.Bundle{} + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ + "schemas": dyn.NewValue(map[string]dyn.Value{ + "kept": dyn.NewValue(map[string]dyn.Value{}, nil), + }, nil), + }, nil)) + }) + require.NoError(t, err) + + tests := []struct { + name string + action deployplan.Action + wantErr string + }{ + { + name: "removed resource recorded as protected", + action: deployplan.Action{ResourceKey: "resources.schemas.removed", ActionType: deployplan.Delete, PreventDestroy: true}, + wantErr: "resources.schemas.removed was removed from the configuration, but its last deployment has lifecycle.prevent_destroy set. To delete it, add it back with lifecycle.prevent_destroy: false and deploy, then remove it again", + }, + { + name: "removed resource recorded as unprotected", + action: deployplan.Action{ResourceKey: "resources.schemas.removed", ActionType: deployplan.Delete}, + }, + { + // bundle destroy plans every resource as a delete carrying the recorded flag. For a + // resource still declared the configuration decides, as before. + name: "declared resource is governed by the configuration", + action: deployplan.Action{ResourceKey: "resources.schemas.kept", ActionType: deployplan.Delete, PreventDestroy: true}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := checkForPreventDestroy(b, []deployplan.Action{tt.action}) + if tt.wantErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tt.wantErr) + } + }) + } +} + func TestCheckForPreventDestroyWhenFirstHasNoPreventDestroy(t *testing.T) { b := &bundle.Bundle{} err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {