From fa73994764a808791e2337f1e83bb53b0fc5ffa8 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 16:13:43 +0100 Subject: [PATCH 1/9] UC secrets follow a renamed catalog or schema UC moves a secret with its renamed parent, so the planner now re-keys it (update_id, parent_renamed) instead of recreating it, which failed with ALREADY_EXISTS. A secret cannot be renamed itself, so its DoUpdateWithID only re-keys; the adapter check now accepts DoUpdateWithID for a resource whose provided_id_fields a parent rename can change. Co-authored-by: Isaac --- .nextchanges/bundles/secret-parent-rename.md | 1 + .../secrets/parent_rename/databricks.yml.tmpl | 22 + .../secrets/parent_rename/out.test.toml | 2 + .../secrets/parent_rename/output.txt | 440 ++++++++++++++++++ .../resources/secrets/parent_rename/script | 39 ++ .../resources/secrets/parent_rename/test.toml | 7 + bundle/direct/dresources/adapter.go | 6 +- bundle/direct/dresources/configs/secrets.yml | 9 + bundle/direct/dresources/secret.go | 11 + libs/testserver/catalogs.go | 9 + libs/testserver/schemas.go | 9 + 11 files changed, 553 insertions(+), 2 deletions(-) create mode 100644 .nextchanges/bundles/secret-parent-rename.md create mode 100644 acceptance/bundle/resources/secrets/parent_rename/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/secrets/parent_rename/out.test.toml create mode 100644 acceptance/bundle/resources/secrets/parent_rename/output.txt create mode 100644 acceptance/bundle/resources/secrets/parent_rename/script create mode 100644 acceptance/bundle/resources/secrets/parent_rename/test.toml diff --git a/.nextchanges/bundles/secret-parent-rename.md b/.nextchanges/bundles/secret-parent-rename.md new file mode 100644 index 00000000000..45f0144d674 --- /dev/null +++ b/.nextchanges/bundles/secret-parent-rename.md @@ -0,0 +1 @@ +* UC secrets inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. diff --git a/acceptance/bundle/resources/secrets/parent_rename/databricks.yml.tmpl b/acceptance/bundle/resources/secrets/parent_rename/databricks.yml.tmpl new file mode 100644 index 00000000000..98e310a0dc4 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/databricks.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: The secret's value + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + secrets: + secret1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: mysecret + value: ${var.secret_value} + comment: c1 diff --git a/acceptance/bundle/resources/secrets/parent_rename/out.test.toml b/acceptance/bundle/resources/secrets/parent_rename/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/secrets/parent_rename/output.txt b/acceptance/bundle/resources/secrets/parent_rename/output.txt new file mode 100644 index 00000000000..5ba571ac7c1 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/output.txt @@ -0,0 +1,440 @@ + +=== Deploy a catalog with a schema and a secret in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created secrets.secret1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/catalogs", + "body": { + "name": "test_catalog_[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "myschema" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "comment": "c1", + "name": "mysecret", + "schema_name": "myschema", + "value": "secret-value" + } +} + +=== Rename the schema: UC moves the secret with it, so the secret is re-keyed without a request +>>> update_file.py databricks.yml name: myschema name: myschema2 + +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "skip", + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME]", + "name": "test_catalog_[UNIQUE_NAME]", + "owner": "[USERNAME]" + } + }, + "resources.schemas.schema1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema", + "name": "myschema", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + } + } + }, + "resources.secrets.secret1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "comment": "c1", + "name": "mysecret", + "schema_name": "myschema2", + "value": "secret-value" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "comment": "c1", + "create_time": "[SECRET_CREATE_TIME]", + "created_by": "[USERNAME]", + "effective_owner": "[USERNAME]", + "effective_value": "[redacted]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema.mysecret", + "name": "mysecret", + "owner": "[USERNAME]", + "schema_name": "myschema", + "update_time": "[SECRET_CREATE_TIME]", + "value": "[redacted]" + }, + "changes": { + "create_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[SECRET_CREATE_TIME]" + }, + "created_by": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_owner": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_value": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[redacted]" + }, + "full_name": { + "action": "skip", + "reason": "spec:output_only", + "remote": "test_catalog_[UNIQUE_NAME].myschema.mysecret" + }, + "owner": { + "action": "skip", + "reason": "spec:input_only", + "remote": "[USERNAME]" + }, + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + }, + "update_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[SECRET_CREATE_TIME]" + }, + "value": { + "action": "skip", + "reason": "custom", + "old": "", + "new": "[redacted]", + "remote": "[redacted]" + } + } + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Updated secrets.secret1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/test_catalog_[UNIQUE_NAME].myschema", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "myschema2" + } +} + +>>> [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret +{ + "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret", + "comment": "c1", + "create_time": "[SECRET_CREATE_TIME]" +} + +=== Rename the catalog and change the secret's comment: the update reaches the secret at its new path +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +>>> update_file.py databricks.yml comment: c1 comment: c2 + +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "update_id", + "new_state": { + "value": { + "name": "test_catalog_renamed_[UNIQUE_NAME]" + } + }, + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME]", + "name": "test_catalog_[UNIQUE_NAME]", + "owner": "[USERNAME]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + } + }, + "resources.schemas.schema1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "comment": "", + "created_by": "[USERNAME]", + "custom_max_retention_hours": 0, + "full_name": "test_catalog_[UNIQUE_NAME].myschema2", + "name": "myschema2", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "comment": { + "action": "skip", + "reason": "empty", + "remote": "" + }, + "custom_max_retention_hours": { + "action": "skip", + "reason": "empty", + "remote": 0 + } + } + }, + "resources.secrets.secret1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "comment": "c2", + "name": "mysecret", + "schema_name": "myschema2", + "value": "secret-value" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "comment": "c1", + "create_time": "[SECRET_CREATE_TIME]", + "created_by": "[USERNAME]", + "effective_owner": "[USERNAME]", + "effective_value": "[redacted]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret", + "name": "mysecret", + "owner": "[USERNAME]", + "schema_name": "myschema2", + "update_time": "[SECRET_CREATE_TIME]", + "value": "[redacted]" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "comment": { + "action": "update", + "old": "c1", + "new": "c2", + "remote": "c1" + }, + "create_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[SECRET_CREATE_TIME]" + }, + "created_by": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_owner": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_value": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[redacted]" + }, + "full_name": { + "action": "skip", + "reason": "spec:output_only", + "remote": "test_catalog_[UNIQUE_NAME].myschema2.mysecret" + }, + "owner": { + "action": "skip", + "reason": "spec:input_only", + "remote": "[USERNAME]" + }, + "update_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[SECRET_CREATE_TIME]" + }, + "value": { + "action": "skip", + "reason": "custom", + "old": "", + "new": "[redacted]", + "remote": "[redacted]" + } + } + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated catalogs.catalog1 +Updated schemas.schema1 +Updated secrets.secret1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 3 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test_catalog_renamed_[UNIQUE_NAME]" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/secrets/test_catalog_renamed_[UNIQUE_NAME].myschema2.mysecret", + "q": { + "update_mask": "*" + }, + "body": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "comment": "c2", + "name": "mysecret", + "schema_name": "myschema2", + "value": "secret-value" + } +} + +>>> [CLI] secrets-uc get-secret test_catalog_renamed_[UNIQUE_NAME].myschema2.mysecret +{ + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2.mysecret", + "comment": "c2", + "create_time": "[SECRET_CREATE_TIME]" +} + +=== The state tracks the new path and the plan is a no-op +>>> [CLI] bundle summary --output json +{ + "catalogs": { + "catalog1": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "schemas": { + "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema2" + }, + "secrets": { + "secret1": "test_catalog_renamed_[UNIQUE_NAME].myschema2.mysecret" + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.secrets.secret1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/secrets/parent_rename/script b/acceptance/bundle/resources/secrets/parent_rename/script new file mode 100644 index 00000000000..8c5fa6cc7c9 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/script @@ -0,0 +1,39 @@ +envsubst < databricks.yml.tmpl > databricks.yml +export BUNDLE_VAR_secret_value=secret-value + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type +} + +title "Deploy a catalog with a schema and a secret in it" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +add_repl "$($CLI secrets-uc get-secret "test_catalog_${UNIQUE_NAME}.myschema.mysecret" | jq -r .create_time)" SECRET_CREATE_TIME + +title "Rename the schema: UC moves the secret with it, so the secret is re-keyed without a request" +trace update_file.py databricks.yml "name: myschema" "name: myschema2" +plan +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI secrets-uc get-secret "test_catalog_${UNIQUE_NAME}.myschema2.mysecret" | jq '{full_name, comment, create_time}' + +title "Rename the catalog and change the secret's comment: the update reaches the secret at its new path" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ +trace update_file.py databricks.yml "comment: c1" "comment: c2" +plan +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI secrets-uc get-secret "test_catalog_renamed_${UNIQUE_NAME}.myschema2.mysecret" | jq '{full_name, comment, create_time}' + +title "The state tracks the new path and the plan is a no-op" +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/secrets/parent_rename/test.toml b/acceptance/bundle/resources/secrets/parent_rename/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 64c7fc22d8d..2f899e68b39 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -476,8 +476,10 @@ func (a *Adapter) validate() error { if hasUpdateWithIDTrigger && a.doUpdateWithID == nil { return errors.New("resourceConfig has updatable_id_fields but DoUpdateWithID is not implemented") } - if a.doUpdateWithID != nil && !hasUpdateWithIDTrigger { - return errors.New("DoUpdateWithID is implemented but resourceConfig lacks updatable_id_fields") + // A parent's rename also re-keys a resource through its provided_id_fields (see followParentRenames). + canUpdateWithID := hasUpdateWithIDTrigger || (a.resourceConfig != nil && len(a.resourceConfig.ProvidedIDFields) > 0) + if a.doUpdateWithID != nil && !canUpdateWithID { + return errors.New("DoUpdateWithID is implemented but resourceConfig lacks updatable_id_fields and provided_id_fields") } } diff --git a/bundle/direct/dresources/configs/secrets.yml b/bundle/direct/dresources/configs/secrets.yml index 22ca3d7f49f..82b5ace212c 100644 --- a/bundle/direct/dresources/configs/secrets.yml +++ b/bundle/direct/dresources/configs/secrets.yml @@ -1,3 +1,12 @@ +provided_id_fields: + # The ID is the full name. UC cannot rename a secret, but it moves the secret with a renamed + # catalog or schema, which re-keys it (see followParentRenames). + - field: catalog_name + reason: id_field + - field: schema_name + reason: id_field + - field: name + reason: id_field sensitive_fields: - field: value - field: effective_value diff --git a/bundle/direct/dresources/secret.go b/bundle/direct/dresources/secret.go index 2db0a334f82..2a1235de14f 100644 --- a/bundle/direct/dresources/secret.go +++ b/bundle/direct/dresources/secret.go @@ -104,6 +104,17 @@ func (r *ResourceSecret) DoUpdate(ctx context.Context, id string, state *catalog return remapSecretRemote(response), nil } +// DoUpdateWithID re-keys the secret after its catalog or schema is renamed: UC moves the secret +// with its parent. A secret cannot be renamed itself, so this is the only way its ID changes. +func (r *ResourceSecret) DoUpdateWithID(ctx context.Context, _ string, state *catalog.Secret, entry *PlanEntry) (string, *catalog.Secret, error) { + newID := state.CatalogName + "." + state.SchemaName + "." + state.Name + if entry.Changes.OnlyParentRenamed() { + return newID, nil, nil + } + remote, err := r.DoUpdate(ctx, newID, state, entry) + return newID, remote, err +} + // DoDelete deletes the secret. func (r *ResourceSecret) DoDelete(ctx context.Context, id string, _ *catalog.Secret) error { return r.client.SecretsUc.DeleteSecret(ctx, catalog.DeleteSecretRequest{ diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 6e9ed132551..e6885e3bd0b 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -169,6 +169,15 @@ func (s *FakeWorkspace) moveCatalogChildren(oldCatalogName, newCatalogName strin secret.FullName = newCatalogName + "." + secret.SchemaName + "." + secret.Name s.UCSecrets[secret.FullName] = secret } + for key, secret := range s.UCSecrets { + if secret.CatalogName != oldCatalogName { + continue + } + delete(s.UCSecrets, key) + secret.CatalogName = newCatalogName + secret.FullName = newCatalogName + "." + secret.SchemaName + "." + secret.Name + s.UCSecrets[secret.FullName] = secret + } } // CatalogsDelete mirrors UC, where deleting a catalog with force deletes its schemas, volumes, registered models and secrets too. diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index 78fdd13903f..7e8925b2fc3 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -143,6 +143,15 @@ func (s *FakeWorkspace) moveSchemaChildren(catalogName, oldSchemaName, newSchema secret.FullName = catalogName + "." + newSchemaName + "." + secret.Name s.UCSecrets[secret.FullName] = secret } + for key, secret := range s.UCSecrets { + if secret.CatalogName != catalogName || secret.SchemaName != oldSchemaName { + continue + } + delete(s.UCSecrets, key) + secret.SchemaName = newSchemaName + secret.FullName = catalogName + "." + newSchemaName + "." + secret.Name + s.UCSecrets[secret.FullName] = secret + } } // SchemasDelete mirrors UC, where deleting a schema with force deletes the volumes, registered models and secrets in it too. From 3dc7c2b1d092cead6f53d4fc5fc3883b7d67fa2a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 6 Nov 2026 09:50:38 +0100 Subject: [PATCH 2/9] Link PR in changelog fragment Co-authored-by: Isaac --- .nextchanges/bundles/secret-parent-rename.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/secret-parent-rename.md b/.nextchanges/bundles/secret-parent-rename.md index 45f0144d674..0d3c9b7d1ce 100644 --- a/.nextchanges/bundles/secret-parent-rename.md +++ b/.nextchanges/bundles/secret-parent-rename.md @@ -1 +1 @@ -* UC secrets inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. +* UC secrets inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. ([#7381](https://github.com/databricks/cli/pull/7381)) From 66d2c5c8751d61c6b02bb6d7e63debc88c54d433 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 12:51:46 +0100 Subject: [PATCH 3/9] acc: show the skipped schema_id change in the secret parent-rename plans Co-authored-by: Isaac --- .../resources/secrets/parent_rename/output.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/acceptance/bundle/resources/secrets/parent_rename/output.txt b/acceptance/bundle/resources/secrets/parent_rename/output.txt index 5ba571ac7c1..13db17659c0 100644 --- a/acceptance/bundle/resources/secrets/parent_rename/output.txt +++ b/acceptance/bundle/resources/secrets/parent_rename/output.txt @@ -81,6 +81,12 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "old": "myschema", "new": "myschema2", "remote": "myschema" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" } } }, @@ -271,6 +277,12 @@ Resources: 0 created, 2 changed, 0 deleted, 1 unchanged "action": "skip", "reason": "empty", "remote": 0 + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" } } }, From e09f4d9ed3becfc1a85561f79ab15f3d20df4338 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 14:10:28 +0100 Subject: [PATCH 4/9] Point comments at followsParentRename Co-authored-by: Isaac --- bundle/direct/dresources/adapter.go | 2 +- bundle/direct/dresources/configs/secrets.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 2f899e68b39..5b303c1099a 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -476,7 +476,7 @@ func (a *Adapter) validate() error { if hasUpdateWithIDTrigger && a.doUpdateWithID == nil { return errors.New("resourceConfig has updatable_id_fields but DoUpdateWithID is not implemented") } - // A parent's rename also re-keys a resource through its provided_id_fields (see followParentRenames). + // A parent's rename also re-keys a resource through its provided_id_fields (see followsParentRename). canUpdateWithID := hasUpdateWithIDTrigger || (a.resourceConfig != nil && len(a.resourceConfig.ProvidedIDFields) > 0) if a.doUpdateWithID != nil && !canUpdateWithID { return errors.New("DoUpdateWithID is implemented but resourceConfig lacks updatable_id_fields and provided_id_fields") diff --git a/bundle/direct/dresources/configs/secrets.yml b/bundle/direct/dresources/configs/secrets.yml index 82b5ace212c..6ef5965522a 100644 --- a/bundle/direct/dresources/configs/secrets.yml +++ b/bundle/direct/dresources/configs/secrets.yml @@ -1,6 +1,6 @@ provided_id_fields: # The ID is the full name. UC cannot rename a secret, but it moves the secret with a renamed - # catalog or schema, which re-keys it (see followParentRenames). + # catalog or schema, which re-keys it (see followsParentRename). - field: catalog_name reason: id_field - field: schema_name From 9d66f2730c59a6a88231935512c27b5c9e4b9fa1 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 15:51:45 +0100 Subject: [PATCH 5/9] direct: let the framework re-key a secret that only follows its parent; drop duplicated test-server moves Co-authored-by: Isaac --- .../resources/secrets/parent_rename/output.txt | 8 ++++---- bundle/direct/dresources/secret.go | 15 +++++++++------ libs/testserver/catalogs.go | 9 --------- libs/testserver/schemas.go | 9 --------- 4 files changed, 13 insertions(+), 28 deletions(-) diff --git a/acceptance/bundle/resources/secrets/parent_rename/output.txt b/acceptance/bundle/resources/secrets/parent_rename/output.txt index 13db17659c0..929a3b532dd 100644 --- a/acceptance/bundle/resources/secrets/parent_rename/output.txt +++ b/acceptance/bundle/resources/secrets/parent_rename/output.txt @@ -102,6 +102,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } ], "action": "update_id", + "state_only": true, "new_state": { "value": { "catalog_name": "test_catalog_[UNIQUE_NAME]", @@ -182,9 +183,8 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated schemas.schema1 -Updated secrets.secret1 Files: 1 uploaded, 0 deleted -Resources: 0 created, 2 changed, 0 deleted, 1 unchanged +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged >>> print_requests.py //unity-catalog { @@ -243,6 +243,7 @@ Resources: 0 created, 2 changed, 0 deleted, 1 unchanged } ], "action": "update_id", + "state_only": true, "new_state": { "value": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", @@ -384,10 +385,9 @@ Resources: 0 created, 2 changed, 0 deleted, 1 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated catalogs.catalog1 -Updated schemas.schema1 Updated secrets.secret1 Files: 1 uploaded, 0 deleted -Resources: 0 created, 3 changed, 0 deleted, 0 unchanged +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged >>> print_requests.py //unity-catalog { diff --git a/bundle/direct/dresources/secret.go b/bundle/direct/dresources/secret.go index 2a1235de14f..38edb0265eb 100644 --- a/bundle/direct/dresources/secret.go +++ b/bundle/direct/dresources/secret.go @@ -104,13 +104,16 @@ func (r *ResourceSecret) DoUpdate(ctx context.Context, id string, state *catalog return remapSecretRemote(response), nil } -// DoUpdateWithID re-keys the secret after its catalog or schema is renamed: UC moves the secret -// with its parent. A secret cannot be renamed itself, so this is the only way its ID changes. +// IDFromState returns the full name, which is the ID. +func (*ResourceSecret) IDFromState(state *catalog.Secret) string { + return state.CatalogName + "." + state.SchemaName + "." + state.Name +} + +// DoUpdateWithID updates a secret whose catalog or schema was renamed: UC moved the secret with its +// parent, so it is addressed at its new ID. A secret cannot be renamed itself, so this is the only +// way its ID changes. func (r *ResourceSecret) DoUpdateWithID(ctx context.Context, _ string, state *catalog.Secret, entry *PlanEntry) (string, *catalog.Secret, error) { - newID := state.CatalogName + "." + state.SchemaName + "." + state.Name - if entry.Changes.OnlyParentRenamed() { - return newID, nil, nil - } + newID := r.IDFromState(state) remote, err := r.DoUpdate(ctx, newID, state, entry) return newID, remote, err } diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index e6885e3bd0b..6e9ed132551 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -169,15 +169,6 @@ func (s *FakeWorkspace) moveCatalogChildren(oldCatalogName, newCatalogName strin secret.FullName = newCatalogName + "." + secret.SchemaName + "." + secret.Name s.UCSecrets[secret.FullName] = secret } - for key, secret := range s.UCSecrets { - if secret.CatalogName != oldCatalogName { - continue - } - delete(s.UCSecrets, key) - secret.CatalogName = newCatalogName - secret.FullName = newCatalogName + "." + secret.SchemaName + "." + secret.Name - s.UCSecrets[secret.FullName] = secret - } } // CatalogsDelete mirrors UC, where deleting a catalog with force deletes its schemas, volumes, registered models and secrets too. diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index 7e8925b2fc3..78fdd13903f 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -143,15 +143,6 @@ func (s *FakeWorkspace) moveSchemaChildren(catalogName, oldSchemaName, newSchema secret.FullName = catalogName + "." + newSchemaName + "." + secret.Name s.UCSecrets[secret.FullName] = secret } - for key, secret := range s.UCSecrets { - if secret.CatalogName != catalogName || secret.SchemaName != oldSchemaName { - continue - } - delete(s.UCSecrets, key) - secret.SchemaName = newSchemaName - secret.FullName = catalogName + "." + newSchemaName + "." + secret.Name - s.UCSecrets[secret.FullName] = secret - } } // SchemasDelete mirrors UC, where deleting a schema with force deletes the volumes, registered models and secrets in it too. From 39a3d65b20951eadcc2146f982750740d1b37f39 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 15:10:43 +0100 Subject: [PATCH 6/9] acc: a secret recreated while its schema is renamed (Badness) The recreate deletes the secret at its recorded path after the schema's rename moved it, so the moved secret is left behind. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 21 ++ .../recreate_in_renamed_schema/out.test.toml | 2 + .../recreate_in_renamed_schema/output.txt | 221 ++++++++++++++++++ .../secrets/recreate_in_renamed_schema/script | 38 +++ .../recreate_in_renamed_schema/test.toml | 8 + 5 files changed, 290 insertions(+) create mode 100644 acceptance/bundle/resources/secrets/recreate_in_renamed_schema/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml create mode 100644 acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt create mode 100644 acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script create mode 100644 acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/databricks.yml.tmpl b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/databricks.yml.tmpl new file mode 100644 index 00000000000..30a41a92d18 --- /dev/null +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/databricks.yml.tmpl @@ -0,0 +1,21 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: The secret's value + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + secrets: + secret1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: mysecret + value: ${var.secret_value} diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt new file mode 100644 index 00000000000..0b09e17af8d --- /dev/null +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt @@ -0,0 +1,221 @@ + +=== Deploy a catalog with a schema and a secret in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created secrets.secret1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema and the secret in one deploy. UC cannot rename a secret, so the secret is recreated +>>> update_file.py databricks.yml name: myschema name: myschema2 + +>>> update_file.py databricks.yml name: mysecret name: mysecret2 + +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "skip", + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME]", + "name": "test_catalog_[UNIQUE_NAME]", + "owner": "[USERNAME]" + } + }, + "resources.schemas.schema1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema", + "name": "myschema", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } + } + }, + "resources.secrets.secret1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "recreate", + "new_state": { + "value": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "mysecret2", + "schema_name": "myschema2", + "value": "secret-value" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "create_time": "[OLD_SECRET_CREATE_TIME]", + "created_by": "[USERNAME]", + "effective_owner": "[USERNAME]", + "effective_value": "[redacted]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema.mysecret", + "name": "mysecret", + "owner": "[USERNAME]", + "schema_name": "myschema", + "update_time": "[OLD_SECRET_CREATE_TIME]", + "value": "[redacted]" + }, + "changes": { + "create_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[OLD_SECRET_CREATE_TIME]" + }, + "created_by": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_owner": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_value": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[redacted]" + }, + "full_name": { + "action": "skip", + "reason": "spec:output_only", + "remote": "test_catalog_[UNIQUE_NAME].myschema.mysecret" + }, + "name": { + "action": "recreate", + "reason": "id_field", + "old": "mysecret", + "new": "mysecret2", + "remote": "mysecret" + }, + "owner": { + "action": "skip", + "reason": "spec:input_only", + "remote": "[USERNAME]" + }, + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + }, + "update_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[OLD_SECRET_CREATE_TIME]" + }, + "value": { + "action": "skip", + "reason": "custom", + "old": "", + "new": "[redacted]", + "remote": "[redacted]" + } + } + } +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Recreated secrets.secret1 +Updated schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 1 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/test_catalog_[UNIQUE_NAME].myschema", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "myschema2" + } +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/secrets/test_catalog_[UNIQUE_NAME].myschema.mysecret" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "mysecret2", + "schema_name": "myschema2", + "value": "secret-value" + } +} + +=== The new secret is in the renamed schema +>>> [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret2 +{ + "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret2" +} + +=== The old secret moved with the renamed schema. It should have been deleted +>>> [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret +{ + "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret", + "create_time": "[OLD_SECRET_CREATE_TIME]" +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.secrets.secret1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script new file mode 100644 index 00000000000..0c463bbe4d1 --- /dev/null +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script @@ -0,0 +1,38 @@ +envsubst < databricks.yml.tmpl > databricks.yml +export BUNDLE_VAR_secret_value=secret-value + +CATALOG="test_catalog_${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type +} + +title "Deploy a catalog with a schema and a secret in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI secrets-uc get-secret "$CATALOG.myschema.mysecret" | jq -r .create_time)" OLD_SECRET_CREATE_TIME + +title "Rename the schema and the secret in one deploy. UC cannot rename a secret, so the secret is recreated" +trace update_file.py databricks.yml "name: myschema" "name: myschema2" +trace update_file.py databricks.yml "name: mysecret" "name: mysecret2" +plan +trace $CLI bundle deploy --auto-approve +trace print_requests.py //unity-catalog + +title "The new secret is in the renamed schema" +trace $CLI secrets-uc get-secret "$CATALOG.myschema2.mysecret2" | jq '{full_name}' + +title "The old secret moved with the renamed schema. It should have been deleted" +trace $CLI secrets-uc get-secret "$CATALOG.myschema2.mysecret" | jq '{full_name, create_time}' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml new file mode 100644 index 00000000000..ce926750f86 --- /dev/null +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml @@ -0,0 +1,8 @@ +Badness = "The schema's rename moves the secret, but the recreate deletes the secret at its recorded path, which no longer exists. The secret moved with the schema is left behind untracked." +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] From b3a8a6410606a3d4f3cd33f82561594a6652f04b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 16:19:16 +0100 Subject: [PATCH 7/9] acc: the secret recreated in a renamed schema is deleted where it moved Co-authored-by: Isaac --- .../secrets/recreate_in_renamed_schema/output.txt | 12 +++++------- .../secrets/recreate_in_renamed_schema/script | 4 ++-- .../secrets/recreate_in_renamed_schema/test.toml | 1 - 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt index 0b09e17af8d..b3dc6b154bb 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt @@ -65,6 +65,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } }, "resources.secrets.secret1": { + "id": "test_catalog_[UNIQUE_NAME].myschema2.mysecret", "depends_on": [ { "node": "resources.catalogs.catalog1", @@ -177,7 +178,7 @@ Resources: 1 created, 1 changed, 1 deleted, 1 unchanged } { "method": "DELETE", - "path": "/api/2.1/unity-catalog/secrets/test_catalog_[UNIQUE_NAME].myschema.mysecret" + "path": "/api/2.1/unity-catalog/secrets/test_catalog_[UNIQUE_NAME].myschema2.mysecret" } { "method": "POST", @@ -196,12 +197,9 @@ Resources: 1 created, 1 changed, 1 deleted, 1 unchanged "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret2" } -=== The old secret moved with the renamed schema. It should have been deleted ->>> [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret -{ - "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret", - "create_time": "[OLD_SECRET_CREATE_TIME]" -} +=== The old secret, moved with the renamed schema, was deleted there +>>> musterr [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret +Error: Secret test_catalog_[UNIQUE_NAME].myschema2.mysecret not found === Redeploy is a no-op >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script index 0c463bbe4d1..cfe49f51a49 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script @@ -31,8 +31,8 @@ trace print_requests.py //unity-catalog title "The new secret is in the renamed schema" trace $CLI secrets-uc get-secret "$CATALOG.myschema2.mysecret2" | jq '{full_name}' -title "The old secret moved with the renamed schema. It should have been deleted" -trace $CLI secrets-uc get-secret "$CATALOG.myschema2.mysecret" | jq '{full_name, create_time}' +title "The old secret, moved with the renamed schema, was deleted there" +trace musterr $CLI secrets-uc get-secret "$CATALOG.myschema2.mysecret" title "Redeploy is a no-op" trace $CLI bundle plan diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml index ce926750f86..4272a80cf61 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml @@ -1,4 +1,3 @@ -Badness = "The schema's rename moves the secret, but the recreate deletes the secret at its recorded path, which no longer exists. The secret moved with the schema is left behind untracked." Cloud = true Ignore = [ From 9ca83b45c15b07f11bd1c47e67c6dc908fe05fe7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 16:25:05 +0100 Subject: [PATCH 8/9] testserver: report a missing UC secret with UC's message Co-authored-by: Isaac --- .../resources/secrets/recreate_in_renamed_schema/output.txt | 2 +- libs/testserver/uc_secrets.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt index b3dc6b154bb..e8cf5a3cbcb 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt @@ -199,7 +199,7 @@ Resources: 1 created, 1 changed, 1 deleted, 1 unchanged === The old secret, moved with the renamed schema, was deleted there >>> musterr [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret -Error: Secret test_catalog_[UNIQUE_NAME].myschema2.mysecret not found +Error: Secret 'test_catalog_[UNIQUE_NAME].myschema2.mysecret' does not exist. === Redeploy is a no-op >>> [CLI] bundle plan diff --git a/libs/testserver/uc_secrets.go b/libs/testserver/uc_secrets.go index 28bacf1fbed..425cffdb1ba 100644 --- a/libs/testserver/uc_secrets.go +++ b/libs/testserver/uc_secrets.go @@ -94,7 +94,7 @@ func (s *FakeWorkspace) SecretsUcGetSecret(req Request) Response { StatusCode: 404, Body: map[string]any{ "error_code": "RESOURCE_DOES_NOT_EXIST", - "message": fmt.Sprintf("Secret %s not found", fullName), + "message": fmt.Sprintf("Secret '%s' does not exist.", fullName), }, } } @@ -143,7 +143,7 @@ func (s *FakeWorkspace) SecretsUcUpdateSecret(req Request) Response { StatusCode: 404, Body: map[string]any{ "error_code": "RESOURCE_DOES_NOT_EXIST", - "message": fmt.Sprintf("Secret %s not found", fullName), + "message": fmt.Sprintf("Secret '%s' does not exist.", fullName), }, } } @@ -195,7 +195,7 @@ func (s *FakeWorkspace) SecretsUcDeleteSecret(req Request) Response { StatusCode: 404, Body: map[string]any{ "error_code": "RESOURCE_DOES_NOT_EXIST", - "message": fmt.Sprintf("Secret %s not found", fullName), + "message": fmt.Sprintf("Secret '%s' does not exist.", fullName), }, } } From 21ac5c270fac91952b57fb7e501d74137e98ce18 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 16:44:34 +0100 Subject: [PATCH 9/9] acc: also deploy the secret rename tests from a saved plan (READPLAN) Co-authored-by: Isaac --- .../bundle/resources/secrets/parent_rename/out.test.toml | 1 + acceptance/bundle/resources/secrets/parent_rename/script | 6 +++--- .../bundle/resources/secrets/parent_rename/test.toml | 7 +++++++ .../secrets/recreate_in_renamed_schema/out.test.toml | 1 + .../resources/secrets/recreate_in_renamed_schema/script | 4 ++-- .../resources/secrets/recreate_in_renamed_schema/test.toml | 7 +++++++ 6 files changed, 21 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/secrets/parent_rename/out.test.toml b/acceptance/bundle/resources/secrets/parent_rename/out.test.toml index 0f167effef1..7a62f726f25 100644 --- a/acceptance/bundle/resources/secrets/parent_rename/out.test.toml +++ b/acceptance/bundle/resources/secrets/parent_rename/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/secrets/parent_rename/script b/acceptance/bundle/resources/secrets/parent_rename/script index 8c5fa6cc7c9..efeb9c91c31 100644 --- a/acceptance/bundle/resources/secrets/parent_rename/script +++ b/acceptance/bundle/resources/secrets/parent_rename/script @@ -11,7 +11,7 @@ trap cleanup EXIT # Drop the remote_state fields that differ between clouds or from the test server. plan() { - trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type + trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type } title "Deploy a catalog with a schema and a secret in it" @@ -22,7 +22,7 @@ add_repl "$($CLI secrets-uc get-secret "test_catalog_${UNIQUE_NAME}.myschema.mys title "Rename the schema: UC moves the secret with it, so the secret is re-keyed without a request" trace update_file.py databricks.yml "name: myschema" "name: myschema2" plan -trace $CLI bundle deploy +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) trace print_requests.py //unity-catalog trace $CLI secrets-uc get-secret "test_catalog_${UNIQUE_NAME}.myschema2.mysecret" | jq '{full_name, comment, create_time}' @@ -30,7 +30,7 @@ title "Rename the catalog and change the secret's comment: the update reaches th trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ trace update_file.py databricks.yml "comment: c1" "comment: c2" plan -trace $CLI bundle deploy +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) trace print_requests.py //unity-catalog trace $CLI secrets-uc get-secret "test_catalog_renamed_${UNIQUE_NAME}.myschema2.mysecret" | jq '{full_name, comment, create_time}' diff --git a/acceptance/bundle/resources/secrets/parent_rename/test.toml b/acceptance/bundle/resources/secrets/parent_rename/test.toml index 4272a80cf61..ebd892fd9df 100644 --- a/acceptance/bundle/resources/secrets/parent_rename/test.toml +++ b/acceptance/bundle/resources/secrets/parent_rename/test.toml @@ -1,7 +1,14 @@ Cloud = true +EnvMatrix.READPLAN = ["", "1"] Ignore = [ ".databricks", ".gitignore", "databricks.yml", ] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml index 0f167effef1..7a62f726f25 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script index cfe49f51a49..3c3b984a7e5 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script @@ -13,7 +13,7 @@ trap cleanup EXIT # Drop the remote_state fields that differ between clouds or from the test server. plan() { - trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type + trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type } title "Deploy a catalog with a schema and a secret in it" @@ -25,7 +25,7 @@ title "Rename the schema and the secret in one deploy. UC cannot rename a secret trace update_file.py databricks.yml "name: myschema" "name: myschema2" trace update_file.py databricks.yml "name: mysecret" "name: mysecret2" plan -trace $CLI bundle deploy --auto-approve +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) --auto-approve trace print_requests.py //unity-catalog title "The new secret is in the renamed schema" diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml index 4272a80cf61..ebd892fd9df 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml @@ -1,7 +1,14 @@ Cloud = true +EnvMatrix.READPLAN = ["", "1"] Ignore = [ ".databricks", ".gitignore", "databricks.yml", ] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = ''