From fc43083e467eda1fbeca8d36684da2f3a5149103 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 4 Nov 2026 13:44:37 +0100 Subject: [PATCH 01/75] Rename schemas in place in the direct engine Changing a schema's name used to delete and recreate it. It is now an updatable ID field renamed via PATCH new_name, which keeps the schema and everything in it. UC moves the children of a renamed catalog or schema with it. A child volume (or a schema in a renamed catalog) whose catalog_name/schema_name references the parent's name was planned as a recreate, and its create then failed with RESOURCE_ALREADY_EXISTS on every deploy. The planner now re-keys such children (update_id, reason parent_renamed), and they are addressed under their new parents. The test server now moves children on catalog/schema rename and rejects duplicate volume names, matching UC. Co-authored-by: Isaac --- .nextchanges/bundles/schema-rename.md | 1 + .../bundle/lifecycle/prevent-destroy/script | 4 +- .../bundle/migrate/auto/schema-rename/script | 15 +- .../bundle/migrate/auto/volume-rename/script | 2 +- .../catalogs/rename/databricks.yml.tmpl | 16 ++ .../resources/catalogs/rename/out.test.toml | 2 + .../resources/catalogs/rename/output.txt | 176 ++++++++++++++++++ .../bundle/resources/catalogs/rename/script | 47 +++++ .../resources/catalogs/rename/test.toml | 7 + .../schemas/rename/databricks.yml.tmpl | 9 + .../resources/schemas/rename/out.test.toml | 2 + .../resources/schemas/rename/output.txt | 99 ++++++++++ .../bundle/resources/schemas/rename/script | 47 +++++ .../bundle/resources/schemas/rename/test.toml | 7 + .../rename_with_volume/databricks.yml.tmpl | 13 ++ .../schemas/rename_with_volume/out.test.toml | 2 + .../schemas/rename_with_volume/output.txt | 132 +++++++++++++ .../schemas/rename_with_volume/script | 48 +++++ .../schemas/rename_with_volume/test.toml | 7 + .../volumes/uppercase-name/out.deploy.txt | 2 +- bundle/deployplan/plan.go | 4 + bundle/direct/bundle_plan.go | 92 +++++++++ bundle/direct/bundle_plan_test.go | 58 ++++++ bundle/direct/dresources/configs/schemas.yml | 7 +- bundle/direct/dresources/schema.go | 36 +++- bundle/direct/dresources/volume.go | 6 +- libs/testserver/volumes.go | 10 + 27 files changed, 828 insertions(+), 23 deletions(-) create mode 100644 .nextchanges/bundles/schema-rename.md create mode 100644 acceptance/bundle/resources/catalogs/rename/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/catalogs/rename/out.test.toml create mode 100644 acceptance/bundle/resources/catalogs/rename/output.txt create mode 100644 acceptance/bundle/resources/catalogs/rename/script create mode 100644 acceptance/bundle/resources/catalogs/rename/test.toml create mode 100644 acceptance/bundle/resources/schemas/rename/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/rename/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/rename/output.txt create mode 100644 acceptance/bundle/resources/schemas/rename/script create mode 100644 acceptance/bundle/resources/schemas/rename/test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume/output.txt create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume/script create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume/test.toml diff --git a/.nextchanges/bundles/schema-rename.md b/.nextchanges/bundles/schema-rename.md new file mode 100644 index 00000000000..bc30b0568b4 --- /dev/null +++ b/.nextchanges/bundles/schema-rename.md @@ -0,0 +1 @@ +* Rename a schema in place when its `name` changes, instead of deleting and recreating it (direct engine). Schemas and volumes inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. ([#PR](https://github.com/databricks/cli/pull/PR)) diff --git a/acceptance/bundle/lifecycle/prevent-destroy/script b/acceptance/bundle/lifecycle/prevent-destroy/script index a2c90cea5b5..94c074cfc0b 100644 --- a/acceptance/bundle/lifecycle/prevent-destroy/script +++ b/acceptance/bundle/lifecycle/prevent-destroy/script @@ -12,8 +12,8 @@ update_file.py resources/pipeline.yml ' catalog: main' ' storage: "dbf trace errcode $CLI bundle plan >>out.txt 2>&1 trace musterr $CLI bundle deploy -qq >>out.txt 2>&1 -# Changing the schema name, deploy must fail because schema will be recreated -update_file.py resources/schema.yml 'name: test-schema' 'name: test-schema-new' +# Changing the schema catalog, deploy must fail because schema will be recreated +update_file.py resources/schema.yml 'catalog_name: "test-catalog"' 'catalog_name: "test-catalog-new"' trace errcode $CLI bundle plan >>out.txt 2>&1 trace musterr $CLI bundle deploy -qq >>out.txt 2>&1 diff --git a/acceptance/bundle/migrate/auto/schema-rename/script b/acceptance/bundle/migrate/auto/schema-rename/script index 2286ccca1b2..273c43e1917 100644 --- a/acceptance/bundle/migrate/auto/schema-rename/script +++ b/acceptance/bundle/migrate/auto/schema-rename/script @@ -10,17 +10,12 @@ title "Replay the prior terraform deployment (captured from a real terraform run trace replay_tfstate.py terraform-requests.json terraform.tfstate rm -f "$OUT_REQUESTS" -title "Rename the schema (an immutable id field). Without --auto-approve the recreate is refused" -# name is a provided id field, so a rename recreates. reconcileIDFields seeds the deployed -# name into the converted state, so the migration's plan sees a real recreate. Schemas are in -# the destructive-approval group, so without --auto-approve the deploy refuses the recreate -# (exit 1) and the migration is not committed - the run stays on terraform. +title "Rename the schema (an updatable id field) and deploy on direct" +# name is an updatable id field: the schema is renamed in place (UpdateWithID), so the deploy +# needs no --auto-approve. reconcileIDFields seeds the deployed name into the converted state, +# so the migration's plan sees a rename rather than snapshotting the new name as already applied. update_file.py databricks.yml "myschema-$UNIQUE_NAME" "myschema-$UNIQUE_NAME-renamed" -trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle deploy -trace print_migration_telemetry - -title "With --auto-approve the recreate is applied on direct under the new name" -trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy --auto-approve +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy title "Redeploy on direct: the schema now matches config, so the plan is a clean no-op" trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy diff --git a/acceptance/bundle/migrate/auto/volume-rename/script b/acceptance/bundle/migrate/auto/volume-rename/script index 5ef772365b7..b4d3abb6b3b 100644 --- a/acceptance/bundle/migrate/auto/volume-rename/script +++ b/acceptance/bundle/migrate/auto/volume-rename/script @@ -10,7 +10,7 @@ title "Replay the prior terraform deployment (captured from a real terraform run trace replay_tfstate.py terraform-requests.json terraform.tfstate title "Rename the volume (an updatable id field) and deploy on direct" -# Unlike a schema (whose name recreates), a volume name is an updatable id field: it is +# A volume name is an updatable id field: it is # renamed in place (UpdateWithID). The migration seeds the deployed name into the converted # state, so its plan sees a rename and the migration succeeds; the deploy then renames the # volume rather than the migration snapshotting the new name as already applied. Verifies the diff --git a/acceptance/bundle/resources/catalogs/rename/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/rename/databricks.yml.tmpl new file mode 100644 index 00000000000..e45eccddc44 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + volumes: + volume1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/catalogs/rename/out.test.toml b/acceptance/bundle/resources/catalogs/rename/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt new file mode 100644 index 00000000000..a4a6e81a101 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -0,0 +1,176 @@ + +=== Deploy a catalog with a schema and a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/catalogs", + "body": { + "name": "test_catalog_[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "myschema" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "myvolume", + "schema_name": "myschema", + "volume_type": "MANAGED" + } +} + +=== Rename the catalog +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +=== UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them +>>> [CLI] bundle plan +update catalogs.catalog1 +update schemas.schema1 +update volumes.volume1 + +Plan: 0 to add, 3 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + }, + "resources.schemas.schema1": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + }, + "resources.volumes.volume1": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + } +} + +=== Deploy renames the catalog and updates the schema and the volume at their new paths +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated catalogs.catalog1 +Updated schemas.schema1 +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 3 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test_catalog_renamed_[UNIQUE_NAME]" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/test_catalog_renamed_[UNIQUE_NAME].myschema", + "body": { + "comment": "", + "custom_max_retention_hours": 0 + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", + "body": { + "comment": "" + } +} + +=== The old catalog is gone and the schema and the volume are under the new one +old catalog is gone + +>>> [CLI] schemas get test_catalog_renamed_[UNIQUE_NAME].myschema +{ + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema", + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]" +} + +>>> [CLI] volumes read test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume +{ + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]" +} + +=== The schema_id and volume_id are preserved, so these are the same objects +schema_id is unchanged +volume_id is unchanged + +=== The state tracks the new paths +>>> [CLI] bundle summary --output json +{ + "catalogs": { + "catalog1": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "schemas": { + "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema" + }, + "volumes": { + "volume1": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume" + } +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 3 unchanged + +>>> print_requests.py //unity-catalog + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script new file mode 100644 index 00000000000..d9421c00eb8 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -0,0 +1,47 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_CATALOG="test_catalog_${UNIQUE_NAME}" +NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a catalog with a schema and a volume in it" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +SCHEMA_ID=$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id) +VOLUME_ID=$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id) + +title "Rename the catalog" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +title "UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them" +trace $CLI bundle plan +trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values(.changes | with_entries(select(.value.action != "skip")))' + +title "Deploy renames the catalog and updates the schema and the volume at their new paths" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "The old catalog is gone and the schema and the volume are under the new one\n" +# Not traced: the "not found" message differs between UC and the test server. +$CLI catalogs get "$OLD_CATALOG" &> /dev/null && echo "old catalog still exists" || echo "old catalog is gone" +trace $CLI schemas get "$NEW_CATALOG.myschema" | jq '{full_name, catalog_name}' +trace $CLI volumes read "$NEW_CATALOG.myschema.myvolume" | jq '{full_name, catalog_name}' + +title "The schema_id and volume_id are preserved, so these are the same objects\n" +[ "$($CLI schemas get "$NEW_CATALOG.myschema" | jq -r .schema_id)" = "$SCHEMA_ID" ] && echo "schema_id is unchanged" || echo "schema_id changed" +[ "$($CLI volumes read "$NEW_CATALOG.myschema.myvolume" | jq -r .volume_id)" = "$VOLUME_ID" ] && echo "volume_id is unchanged" || echo "volume_id changed" + +title "The state tracks the new paths" +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' + +title "Redeploy is a no-op" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/catalogs/rename/test.toml b/acceptance/bundle/resources/catalogs/rename/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/rename/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename/databricks.yml.tmpl new file mode 100644 index 00000000000..080c703a351 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + comment: COMMENT1 diff --git a/acceptance/bundle/resources/schemas/rename/out.test.toml b/acceptance/bundle/resources/schemas/rename/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename/output.txt b/acceptance/bundle/resources/schemas/rename/output.txt new file mode 100644 index 00000000000..a213c84394b --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename/output.txt @@ -0,0 +1,99 @@ + +=== Deploy the schema +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "comment": "COMMENT1", + "name": "test-schema-[UNIQUE_NAME]" + } +} + +=== Rename the schema +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +=== The plan renames the schema in place instead of recreating it +>>> [CLI] bundle plan +update schemas.schema1 + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle plan --output json +{ + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + } +} + +=== Deploy renames the schema without a deletion prompt +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "body": { + "comment": "COMMENT1", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} + +=== The schema is only reachable under the new name +>>> musterr [CLI] schemas get main.test-schema-[UNIQUE_NAME] +Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. + +>>> [CLI] schemas get main.test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "name": "test-schema-renamed-[UNIQUE_NAME]", + "comment": "COMMENT1" +} + +=== The schema_id is preserved, so this is the same schema +schema_id is unchanged + +=== The state and summary track the new name +>>> [CLI] bundle summary --output json +{ + "id": "main.test-schema-renamed-[UNIQUE_NAME]", + "name": "test-schema-renamed-[UNIQUE_NAME]" +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/schemas/rename/script b/acceptance/bundle/resources/schemas/rename/script new file mode 100644 index 00000000000..24dacb067dc --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename/script @@ -0,0 +1,47 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_SCHEMA="main.test-schema-${UNIQUE_NAME}" +NEW_SCHEMA="main.test-schema-renamed-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy the schema" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +SCHEMA_ID=$($CLI schemas get "$OLD_SCHEMA" | jq -r .schema_id) + +title "Rename the schema" +trace update_file.py databricks.yml test-schema- test-schema-renamed- + +title "The plan renames the schema in place instead of recreating it" +trace $CLI bundle plan +trace $CLI bundle plan --output json | nostamp | jq '.plan[].changes' | normalize_uc_payload.py + +title "Deploy renames the schema without a deletion prompt" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "The schema is only reachable under the new name" +trace musterr $CLI schemas get "$OLD_SCHEMA" +trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, name, comment}' + +title "The schema_id is preserved, so this is the same schema\n" +if [ "$($CLI schemas get "$NEW_SCHEMA" | jq -r .schema_id)" = "$SCHEMA_ID" ]; then + echo "schema_id is unchanged" +else + echo "schema_id changed" +fi + +title "The state and summary track the new name" +trace $CLI bundle summary --output json | jq '.resources.schemas.schema1 | {id, name}' + +title "Redeploy is a no-op" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/schemas/rename/test.toml b/acceptance/bundle/resources/schemas/rename/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_with_volume/databricks.yml.tmpl new file mode 100644 index 00000000000..2234fdf2e39 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/out.test.toml b/acceptance/bundle/resources/schemas/rename_with_volume/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt new file mode 100644 index 00000000000..17f9535e737 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -0,0 +1,132 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "test-schema-[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +=== Rename the schema +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +=== UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it +>>> [CLI] bundle plan +update schemas.schema1 +update volumes.volume1 + +Plan: 0 to add, 2 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + } + }, + "resources.volumes.volume1": { + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + } + } +} + +=== Deploy renames the schema and updates the volume at its new path +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "body": { + "comment": "" + } +} + +=== The old schema is gone and the volume is under the new one +>>> musterr [CLI] schemas get main.test-schema-[UNIQUE_NAME] +Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. + +>>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]" +} + +=== The volume_id is preserved, so this is the same volume +volume_id is unchanged + +=== The state tracks the new path +>>> [CLI] bundle summary --output json +"main.test-schema-renamed-[UNIQUE_NAME].myvolume" + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/script b/acceptance/bundle/resources/schemas/rename_with_volume/script new file mode 100644 index 00000000000..478233c03aa --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume/script @@ -0,0 +1,48 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_SCHEMA="main.test-schema-${UNIQUE_NAME}" +OLD_VOLUME="$OLD_SCHEMA.myvolume" +NEW_VOLUME="main.test-schema-renamed-${UNIQUE_NAME}.myvolume" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +VOLUME_ID=$($CLI volumes read "$OLD_VOLUME" | jq -r .volume_id) + +title "Rename the schema" +trace update_file.py databricks.yml test-schema- test-schema-renamed- + +title "UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it" +trace $CLI bundle plan +trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values(.changes | with_entries(select(.value.action != "skip")))' + +title "Deploy renames the schema and updates the volume at its new path" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "The old schema is gone and the volume is under the new one" +trace musterr $CLI schemas get "$OLD_SCHEMA" +trace $CLI volumes read "$NEW_VOLUME" | jq '{full_name, schema_name}' + +title "The volume_id is preserved, so this is the same volume\n" +if [ "$($CLI volumes read "$NEW_VOLUME" | jq -r .volume_id)" = "$VOLUME_ID" ]; then + echo "volume_id is unchanged" +else + echo "volume_id changed" +fi + +title "The state tracks the new path" +trace $CLI bundle summary --output json | jq '.resources.volumes.volume1.id' + +title "Redeploy is a no-op" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/test.toml b/acceptance/bundle/resources/schemas/rename_with_volume/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.txt b/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.txt index bba6a759059..2b50dcb8cfa 100644 --- a/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.txt +++ b/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.txt @@ -33,7 +33,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "changes": { "name": { "action": "skip", - "reason": "id_field" + "reason": "id_changes" } } }, diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 8446b200b12..d53de5eee48 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -184,6 +184,10 @@ const ( // it and whatever the remote reports is not drift (e.g. cluster libraries without a libraries // section, where job runs install their task libraries cluster-wide). ReasonUnmanaged = "unmanaged" + // ReasonParentRenamed: an ID field that references the name of a parent being renamed in + // place. The backend moves the resource along with its parent, so it is re-keyed rather + // than recreated. + ReasonParentRenamed = "parent_renamed" // Special reason that results in removing this change from the plan ReasonDrop = "!drop" diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 237fcc75521..45bf1bdcce9 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -263,6 +263,13 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return true } + // Collected before resolveReferences, which drops the references it resolves. + parentRenames, err := b.parentRenames(resourceKey) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } + // Process all references in the resource using Refs map // Refs maps path inside resource to references e.g. "${resources.jobs.foo.id} ${resources.jobs.foo.name}" if !b.resolveReferences(ctx, resourceKey, entry, errorPrefix, true) { @@ -370,6 +377,12 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } + err = followParentRenames(adapter, entry.Changes, parentRenames) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: classifying changes: %w", errorPrefix, err)) + return false + } + if remoteState == nil { // Even if local action is "recreate" which is higher than "create", we should still pick "create" here // because we know remote does not exist. @@ -666,6 +679,85 @@ func classifyIDField(cfg *dresources.ResourceLifecycleConfig, path *structpath.P return deployplan.Undefined, "", false } +// parentRenames returns the fields of resourceKey whose whole value is a reference to a +// field that the referenced resource renames in this plan (an update_id change), keyed by +// field path, with the referenced change as the value. +func (b *DeploymentBundle) parentRenames(resourceKey string) (map[string]*deployplan.ChangeDesc, error) { + sv, ok := b.StateCache.Load(resourceKey) + if !ok { + return nil, fmt.Errorf("internal error: no state cache entry found for %q", resourceKey) + } + + result := make(map[string]*deployplan.ChangeDesc) + for fieldPath, refString := range sv.Refs { + ref, ok := dynvar.NewRef(dyn.V(refString)) + if !ok || !ref.IsPure() { + continue + } + targetPath, err := structpath.ParsePath(ref.References()[0]) + if err != nil { + return nil, fmt.Errorf("cannot parse reference %q: %w", refString, err) + } + targetKey, targetField := splitResourcePath(targetPath) + change, err := b.renameChange(targetKey, targetField.String()) + if err != nil { + return nil, err + } + if change != nil { + result[fieldPath] = change + } + } + return result, nil +} + +// renameChange returns the change to field of resourceKey if it is part of an update_id, nil otherwise. +func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan.ChangeDesc, error) { + entry, err := b.Plan.ReadLockEntry(resourceKey) + if err != nil { + return nil, err + } + defer b.Plan.ReadUnlockEntry(resourceKey) + + // A missing entry is reported by resolveReferences. + if entry == nil || entry.Action != deployplan.UpdateWithID { + return nil, nil + } + change := entry.Changes[field] + if change == nil || change.Action != deployplan.UpdateWithID { + return nil, nil + } + return change, nil +} + +// followParentRenames plans an update_id instead of a recreate for an ID field that changes +// only because it references the name of a parent that is renamed in place. UC moves the +// children of a renamed catalog or schema along with it, so recreating such a child fails +// with "already exists"; its DoUpdateWithID addresses it under the new parent instead. +// +// The old values must match too: a child moved to the renamed parent from somewhere else +// still has to be recreated. +func followParentRenames(adapter *dresources.Adapter, changes deployplan.Changes, parentRenames map[string]*deployplan.ChangeDesc) error { + if !adapter.HasDoUpdateWithID() { + return nil + } + for pathString, parentChange := range parentRenames { + ch := changes[pathString] + if ch == nil || ch.Action != deployplan.Recreate || !structdiff.IsEqual(ch.Old, parentChange.Old) { + continue + } + path, err := structpath.ParsePath(pathString) + if err != nil { + return err + } + if _, ok := findMatchingRule(path, adapter.ResourceConfig().ProvidedIDFields); !ok { + continue + } + ch.Action = deployplan.UpdateWithID + ch.Reason = deployplan.ReasonParentRenamed + } + return nil +} + // shouldSkipNormalized skips a change that is a false diff caused by UC API // normalization: the API strips trailing slashes from storage URLs // (normalize_slash). The direct engine saves local config to state, so without diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 2d051dc21b9..f50de986918 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -271,6 +271,64 @@ func TestRemoteAlreadySetGuards(t *testing.T) { } } +// TestFollowParentRenames pins when an ID field that references a renamed parent is +// re-keyed (update_id) instead of recreated: only a provided_id_field of a resource with +// DoUpdateWithID, and only if it was under that parent before the rename. +func TestFollowParentRenames(t *testing.T) { + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) + + tests := []struct { + name string + resource string + field string + old string + expectedAction deployplan.ActionType + }{ + { + name: "child of the renamed parent follows it", + resource: "volumes", + field: "schema_name", + old: "s_old", + expectedAction: deployplan.UpdateWithID, + }, + { + name: "child moved from another parent is recreated", + resource: "volumes", + field: "schema_name", + old: "other", + expectedAction: deployplan.Recreate, + }, + { + name: "resource without DoUpdateWithID is recreated", + resource: "registered_models", + field: "schema_name", + old: "s_old", + expectedAction: deployplan.Recreate, + }, + { + name: "recreate_on_changes field is recreated", + resource: "volumes", + field: "storage_location", + old: "s_old", + expectedAction: deployplan.Recreate, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + adapter, ok := adapters[tt.resource] + require.True(t, ok) + ch := &deployplan.ChangeDesc{Action: deployplan.Recreate, Reason: "id_field", Old: tt.old, New: "s_new", Remote: tt.old} + parentRenames := map[string]*deployplan.ChangeDesc{ + tt.field: {Action: deployplan.UpdateWithID, Reason: "id_changes", Old: "s_old", New: "s_new", Remote: "s_old"}, + } + require.NoError(t, followParentRenames(adapter, deployplan.Changes{tt.field: ch}, parentRenames)) + assert.Equal(t, tt.expectedAction, ch.Action) + }) + } +} + // TestRecreateBackendDefault pins the implicit backend-default behavior for // recreate_on_changes fields (recreateOrBackendDefault): an immutable field the // config never set (old and new nil) but the backend populated (remote set) is left diff --git a/bundle/direct/dresources/configs/schemas.yml b/bundle/direct/dresources/configs/schemas.yml index 8ef27e71d15..02f68e4afe9 100644 --- a/bundle/direct/dresources/configs/schemas.yml +++ b/bundle/direct/dresources/configs/schemas.yml @@ -1,9 +1,10 @@ provided_id_fields: - # UC lowercases identifier names; remote returns "myschema" for config "MySchema". - - field: name - reason: id_field - field: catalog_name reason: id_field +updatable_id_fields: + # UC lowercases identifier names; remote returns "myschema" for config "MySchema". + - field: name + reason: id_changes recreate_on_changes: - field: storage_root reason: immutable diff --git a/bundle/direct/dresources/schema.go b/bundle/direct/dresources/schema.go index d80dac584bc..9b7bdd43b51 100644 --- a/bundle/direct/dresources/schema.go +++ b/bundle/direct/dresources/schema.go @@ -64,20 +64,22 @@ func (r *ResourceSchema) DoCreate(ctx context.Context, config *catalog.CreateSch // migrate invariant (schema.yml is excluded there, and schema_tf_compat.yml covers migrate instead). var schemaForceSend = []string{"Comment", "CustomMaxRetentionHours"} -// DoUpdate updates the schema in place and returns remote state. -func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *catalog.CreateSchema, _ *PlanEntry) (*catalog.SchemaInfo, error) { - updateRequest := catalog.UpdateSchema{ +func makeUpdateSchema(id, newName string, config *catalog.CreateSchema) catalog.UpdateSchema { + return catalog.UpdateSchema{ Comment: config.Comment, CustomMaxRetentionHours: config.CustomMaxRetentionHours, EnablePredictiveOptimization: "", // Not supported by DABs FullName: id, - NewName: "", // We recreate schemas on name change intentionally. + NewName: newName, Owner: "", // Not supported by DABs Properties: config.Properties, ForceSendFields: utils.FilterFields[catalog.UpdateSchema](append(slices.Clone(schemaForceSend), config.ForceSendFields...), "EnablePredictiveOptimization", "NewName", "Owner"), } +} - response, err := r.client.Schemas.Update(ctx, updateRequest) +// DoUpdate updates the schema in place and returns remote state. +func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *catalog.CreateSchema, _ *PlanEntry) (*catalog.SchemaInfo, error) { + response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(id, "", config)) if err != nil { return nil, err } @@ -89,6 +91,30 @@ func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *catalo return response, nil } +// DoUpdateWithID renames the schema in place when the name changes; the ID is the full name, so it changes too. +func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateSchema, _ *PlanEntry) (string, *catalog.SchemaInfo, error) { + nameFromID, err := getNameFromID(id) + if err != nil { + return "", nil, err + } + + newName := "" + if config.Name != nameFromID { + newName = config.Name + } + + // The catalog comes from config: a renamed catalog moves the schema with it, leaving id + // stale (see followParentRenames). + fullName := config.CatalogName + "." + nameFromID + + response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(fullName, newName, config)) + if err != nil || response == nil { + return "", nil, err + } + + return response.FullName, response, nil +} + func (r *ResourceSchema) DoDelete(ctx context.Context, id string, _ *catalog.CreateSchema) error { return r.client.Schemas.Delete(ctx, catalog.DeleteSchemaRequest{ FullName: id, diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 7cd1eaba1ce..168de23f1c1 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -86,7 +86,7 @@ func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *catalo func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateVolumeRequestContent, _ *PlanEntry) (string, *catalog.VolumeInfo, error) { updateRequest := catalog.UpdateVolumeRequestContent{ Comment: config.Comment, - Name: id, + Name: "", // Initialized below NewName: "", // Initialized below if needed Owner: "", // Not supported by DABs @@ -100,6 +100,10 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * } nameFromID := items[len(items)-1] + // The parents come from config: a renamed catalog or schema moves the volume with it, + // leaving id stale (see followParentRenames). + updateRequest.Name = config.CatalogName + "." + config.SchemaName + "." + nameFromID + if config.Name != nameFromID { updateRequest.NewName = config.Name } diff --git a/libs/testserver/volumes.go b/libs/testserver/volumes.go index 00466f88b6f..8aa01fdddd3 100644 --- a/libs/testserver/volumes.go +++ b/libs/testserver/volumes.go @@ -51,6 +51,16 @@ func (s *FakeWorkspace) VolumesCreate(req Request) Response { defer s.LockUnlock()() + if _, exists := s.Volumes[volume.FullName]; exists { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{ + "error_code": "RESOURCE_ALREADY_EXISTS", + "message": fmt.Sprintf("Volume '%s' already exists", volume.FullName), + }, + } + } + s.Volumes[volume.FullName] = volume // Register the volume path as a directory so that fs cp works without From 956cff0c739386c26432240c7fa750c8d55dbd64 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 4 Nov 2026 13:46:05 +0100 Subject: [PATCH 02/75] Link PR in changelog fragment Co-authored-by: Isaac --- .nextchanges/bundles/schema-rename.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/schema-rename.md b/.nextchanges/bundles/schema-rename.md index bc30b0568b4..517c74045d7 100644 --- a/.nextchanges/bundles/schema-rename.md +++ b/.nextchanges/bundles/schema-rename.md @@ -1 +1 @@ -* Rename a schema in place when its `name` changes, instead of deleting and recreating it (direct engine). Schemas and volumes inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. ([#PR](https://github.com/databricks/cli/pull/PR)) +* Rename a schema in place when its `name` changes, instead of deleting and recreating it (direct engine). Schemas and volumes inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. ([#7343](https://github.com/databricks/cli/pull/7343)) From 6e0fe9bc1aeb57e443533e132163bd879e14538c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 4 Nov 2026 14:00:34 +0100 Subject: [PATCH 03/75] testserver: keep volume create lenient A captured terraform fixture (migrate/command/basic) deploys the same volume name from two targets. The plan goldens already catch a regression to recreate, so the duplicate check is not needed. Co-authored-by: Isaac --- libs/testserver/volumes.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/libs/testserver/volumes.go b/libs/testserver/volumes.go index 8aa01fdddd3..00466f88b6f 100644 --- a/libs/testserver/volumes.go +++ b/libs/testserver/volumes.go @@ -51,16 +51,6 @@ func (s *FakeWorkspace) VolumesCreate(req Request) Response { defer s.LockUnlock()() - if _, exists := s.Volumes[volume.FullName]; exists { - return Response{ - StatusCode: http.StatusBadRequest, - Body: map[string]string{ - "error_code": "RESOURCE_ALREADY_EXISTS", - "message": fmt.Sprintf("Volume '%s' already exists", volume.FullName), - }, - } - } - s.Volumes[volume.FullName] = volume // Register the volume path as a directory so that fs cp works without From 1903d7237c93f7d9c7f71d4e6ce11b1d3656194c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 4 Nov 2026 15:46:05 +0100 Subject: [PATCH 04/75] Drop engine mention from changelog entry Co-authored-by: Isaac --- .nextchanges/bundles/schema-rename.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/schema-rename.md b/.nextchanges/bundles/schema-rename.md index 517c74045d7..2ac01106dbe 100644 --- a/.nextchanges/bundles/schema-rename.md +++ b/.nextchanges/bundles/schema-rename.md @@ -1 +1 @@ -* Rename a schema in place when its `name` changes, instead of deleting and recreating it (direct engine). Schemas and volumes inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. ([#7343](https://github.com/databricks/cli/pull/7343)) +* Rename a schema in place when its `name` changes, instead of deleting and recreating it. Schemas and volumes inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. ([#7343](https://github.com/databricks/cli/pull/7343)) From 80a4f3d7cd2f8c982c6caa0c6671b640af8872e7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 4 Nov 2026 15:52:56 +0100 Subject: [PATCH 05/75] acc: drop text plan where the JSON plan already shows it Also restores the grants/schemas/change_name golden for the in-place rename, which the rebase onto denik/grants-rekey dropped. Co-authored-by: Isaac --- .../resources/catalogs/rename/output.txt | 52 ++++++++++--------- .../bundle/resources/catalogs/rename/script | 3 +- .../resources/schemas/rename/output.txt | 20 +++---- .../bundle/resources/schemas/rename/script | 3 +- .../schemas/rename_with_volume/output.txt | 36 ++++++------- .../schemas/rename_with_volume/script | 3 +- 6 files changed, 58 insertions(+), 59 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index a4a6e81a101..c0b51984440 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -39,40 +39,42 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ === UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them ->>> [CLI] bundle plan -update catalogs.catalog1 -update schemas.schema1 -update volumes.volume1 - -Plan: 0 to add, 3 to change, 0 to delete, 0 unchanged - >>> [CLI] bundle plan --output json { "resources.catalogs.catalog1": { - "name": { - "action": "update_id", - "reason": "id_changes", - "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]", - "remote": "test_catalog_[UNIQUE_NAME]" + "action": "update_id", + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } } }, "resources.schemas.schema1": { - "catalog_name": { - "action": "update_id", - "reason": "parent_renamed", - "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]", - "remote": "test_catalog_[UNIQUE_NAME]" + "action": "update_id", + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } } }, "resources.volumes.volume1": { - "catalog_name": { - "action": "update_id", - "reason": "parent_renamed", - "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]", - "remote": "test_catalog_[UNIQUE_NAME]" + "action": "update_id", + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } } } } diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script index d9421c00eb8..aecf603a512 100644 --- a/acceptance/bundle/resources/catalogs/rename/script +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -21,8 +21,7 @@ title "Rename the catalog" trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ title "UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them" -trace $CLI bundle plan -trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values(.changes | with_entries(select(.value.action != "skip")))' +trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values({action, changes: ((.changes // {}) | with_entries(select(.value.action != "skip")))})' title "Deploy renames the catalog and updates the schema and the volume at their new paths" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/rename/output.txt b/acceptance/bundle/resources/schemas/rename/output.txt index a213c84394b..b2d3dbeb1e4 100644 --- a/acceptance/bundle/resources/schemas/rename/output.txt +++ b/acceptance/bundle/resources/schemas/rename/output.txt @@ -21,19 +21,19 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml test-schema- test-schema-renamed- === The plan renames the schema in place instead of recreating it ->>> [CLI] bundle plan -update schemas.schema1 - -Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged - >>> [CLI] bundle plan --output json { - "name": { + "resources.schemas.schema1": { "action": "update_id", - "reason": "id_changes", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-[UNIQUE_NAME]" + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + } + } } } diff --git a/acceptance/bundle/resources/schemas/rename/script b/acceptance/bundle/resources/schemas/rename/script index 24dacb067dc..1e5044d9ffc 100644 --- a/acceptance/bundle/resources/schemas/rename/script +++ b/acceptance/bundle/resources/schemas/rename/script @@ -20,8 +20,7 @@ title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- title "The plan renames the schema in place instead of recreating it" -trace $CLI bundle plan -trace $CLI bundle plan --output json | nostamp | jq '.plan[].changes' | normalize_uc_payload.py +trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values({action, changes: ((.changes // {}) | with_entries(select(.value.action != "skip")))})' title "Deploy renames the schema without a deletion prompt" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt index 17f9535e737..bb4775ffb6d 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -31,30 +31,30 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml test-schema- test-schema-renamed- === UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it ->>> [CLI] bundle plan -update schemas.schema1 -update volumes.volume1 - -Plan: 0 to add, 2 to change, 0 to delete, 0 unchanged - >>> [CLI] bundle plan --output json { "resources.schemas.schema1": { - "name": { - "action": "update_id", - "reason": "id_changes", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-[UNIQUE_NAME]" + "action": "update_id", + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + } } }, "resources.volumes.volume1": { - "schema_name": { - "action": "update_id", - "reason": "parent_renamed", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-[UNIQUE_NAME]" + "action": "update_id", + "changes": { + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + } } } } diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/script b/acceptance/bundle/resources/schemas/rename_with_volume/script index 478233c03aa..3197694b3a4 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/script +++ b/acceptance/bundle/resources/schemas/rename_with_volume/script @@ -21,8 +21,7 @@ title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- title "UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it" -trace $CLI bundle plan -trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values(.changes | with_entries(select(.value.action != "skip")))' +trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values({action, changes: ((.changes // {}) | with_entries(select(.value.action != "skip")))})' title "Deploy renames the schema and updates the volume at its new path" trace $CLI bundle deploy From 272312778c5620cea6c4b331dd7c96d42944899d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 4 Nov 2026 16:00:17 +0100 Subject: [PATCH 06/75] acc: show only the plan field each rename test is about Also restores the grants/schemas/change_name golden for the in-place rename. Co-authored-by: Isaac --- .../resources/catalogs/rename/output.txt | 41 +++---------------- .../bundle/resources/catalogs/rename/script | 2 +- .../resources/schemas/rename/output.txt | 17 +++----- .../bundle/resources/schemas/rename/script | 2 +- .../schemas/rename_with_volume/output.txt | 29 +++---------- .../schemas/rename_with_volume/script | 2 +- 6 files changed, 18 insertions(+), 75 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index c0b51984440..29b5fe9ef22 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -41,42 +41,11 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them >>> [CLI] bundle plan --output json { - "resources.catalogs.catalog1": { - "action": "update_id", - "changes": { - "name": { - "action": "update_id", - "reason": "id_changes", - "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]", - "remote": "test_catalog_[UNIQUE_NAME]" - } - } - }, - "resources.schemas.schema1": { - "action": "update_id", - "changes": { - "catalog_name": { - "action": "update_id", - "reason": "parent_renamed", - "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]", - "remote": "test_catalog_[UNIQUE_NAME]" - } - } - }, - "resources.volumes.volume1": { - "action": "update_id", - "changes": { - "catalog_name": { - "action": "update_id", - "reason": "parent_renamed", - "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]", - "remote": "test_catalog_[UNIQUE_NAME]" - } - } - } + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" } === Deploy renames the catalog and updates the schema and the volume at their new paths diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script index aecf603a512..fee34a9d751 100644 --- a/acceptance/bundle/resources/catalogs/rename/script +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -21,7 +21,7 @@ title "Rename the catalog" trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ title "UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them" -trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values({action, changes: ((.changes // {}) | with_entries(select(.value.action != "skip")))})' +trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"].changes.catalog_name' title "Deploy renames the catalog and updates the schema and the volume at their new paths" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/rename/output.txt b/acceptance/bundle/resources/schemas/rename/output.txt index b2d3dbeb1e4..253928c8d6c 100644 --- a/acceptance/bundle/resources/schemas/rename/output.txt +++ b/acceptance/bundle/resources/schemas/rename/output.txt @@ -23,18 +23,11 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === The plan renames the schema in place instead of recreating it >>> [CLI] bundle plan --output json { - "resources.schemas.schema1": { - "action": "update_id", - "changes": { - "name": { - "action": "update_id", - "reason": "id_changes", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-[UNIQUE_NAME]" - } - } - } + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" } === Deploy renames the schema without a deletion prompt diff --git a/acceptance/bundle/resources/schemas/rename/script b/acceptance/bundle/resources/schemas/rename/script index 1e5044d9ffc..ec7a4694578 100644 --- a/acceptance/bundle/resources/schemas/rename/script +++ b/acceptance/bundle/resources/schemas/rename/script @@ -20,7 +20,7 @@ title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- title "The plan renames the schema in place instead of recreating it" -trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values({action, changes: ((.changes // {}) | with_entries(select(.value.action != "skip")))})' +trace $CLI bundle plan --output json | jq '.plan["resources.schemas.schema1"].changes.name' title "Deploy renames the schema without a deletion prompt" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt index bb4775ffb6d..0e943e097c3 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -33,30 +33,11 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it >>> [CLI] bundle plan --output json { - "resources.schemas.schema1": { - "action": "update_id", - "changes": { - "name": { - "action": "update_id", - "reason": "id_changes", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-[UNIQUE_NAME]" - } - } - }, - "resources.volumes.volume1": { - "action": "update_id", - "changes": { - "schema_name": { - "action": "update_id", - "reason": "parent_renamed", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-[UNIQUE_NAME]" - } - } - } + "action": "update_id", + "reason": "parent_renamed", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" } === Deploy renames the schema and updates the volume at its new path diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/script b/acceptance/bundle/resources/schemas/rename_with_volume/script index 3197694b3a4..383f22a7c29 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/script +++ b/acceptance/bundle/resources/schemas/rename_with_volume/script @@ -21,7 +21,7 @@ title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- title "UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it" -trace $CLI bundle plan --output json | nostamp | jq '.plan | map_values({action, changes: ((.changes // {}) | with_entries(select(.value.action != "skip")))})' +trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"].changes.schema_name' title "Deploy renames the schema and updates the volume at its new path" trace $CLI bundle deploy From 6bb6f0fa96e5a344addb117e6658d0d163702ee0 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 11:26:00 +0100 Subject: [PATCH 07/75] acc: point migrate recreate comments at schema-catalog-change schema-rename now renames in place, so it no longer covers the schema recreate. Co-authored-by: Isaac --- acceptance/bundle/migrate/auto/schema-storage-change/script | 2 +- acceptance/bundle/migrate/auto/volume-type-change/script | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/migrate/auto/schema-storage-change/script b/acceptance/bundle/migrate/auto/schema-storage-change/script index c0ab15856de..ef97ca99a82 100644 --- a/acceptance/bundle/migrate/auto/schema-storage-change/script +++ b/acceptance/bundle/migrate/auto/schema-storage-change/script @@ -2,7 +2,7 @@ export DATABRICKS_BUNDLE_ENGINE= # Testserver-only: an explicit storage_root needs a registered external location on a real # workspace, so this is not Cloud-enabled. The recreate/refusal class it covers is verified -# on cloud by schema-rename and pipeline-storage-change. +# on cloud by schema-catalog-change and pipeline-storage-change. title "Replay the prior terraform deployment (captured from a real terraform run)" trace replay_tfstate.py terraform-requests.json terraform.tfstate diff --git a/acceptance/bundle/migrate/auto/volume-type-change/script b/acceptance/bundle/migrate/auto/volume-type-change/script index 6b02a55f247..ea5ebdb1499 100644 --- a/acceptance/bundle/migrate/auto/volume-type-change/script +++ b/acceptance/bundle/migrate/auto/volume-type-change/script @@ -2,7 +2,7 @@ export DATABRICKS_BUNDLE_ENGINE= # Testserver-only: changing volume_type to EXTERNAL needs a storage_location backed by a # registered external location on a real workspace, so this is not Cloud-enabled. The -# recreate class it covers is verified on cloud by schema-rename / registered-model-rename. +# recreate class it covers is verified on cloud by schema-catalog-change / registered-model-rename. # volume_type is both a provided id field and a recreate_on_changes field. reconcileIDFields # records the deployed value, so the migration's plan sees a real change and recreates. Volumes From 4e513bcc7af5b17952ccd894a306cebadef323d1 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 11:35:27 +0100 Subject: [PATCH 08/75] acc: show the whole plan in the rename tests Co-authored-by: Isaac --- .../resources/catalogs/rename/output.txt | 111 +++++++++++++++++- .../bundle/resources/catalogs/rename/script | 3 +- .../resources/schemas/rename/output.txt | 45 ++++++- .../bundle/resources/schemas/rename/script | 2 +- .../schemas/rename_with_volume/output.txt | 83 ++++++++++++- .../schemas/rename_with_volume/script | 3 +- 6 files changed, 229 insertions(+), 18 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index 29b5fe9ef22..a0d4de8b2d1 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -41,11 +41,112 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them >>> [CLI] bundle plan --output json { - "action": "update_id", - "reason": "parent_renamed", - "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]", - "remote": "test_catalog_[UNIQUE_NAME]" + "resources.catalogs.catalog1": { + "action": "update_id", + "new_state": { + "value": { + "name": "test_catalog_renamed_[UNIQUE_NAME]" + } + }, + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME]", + "name": "test_catalog_[UNIQUE_NAME]", + "owner": "[USERNAME]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + } + }, + "resources.schemas.schema1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myschema" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema", + "name": "myschema", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myvolume", + "schema_name": "myschema", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "created_at": [UNIX_TIME_MILLIS][2], + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema.myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "myschema", + "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", + "volume_id": "[UUID]", + "volume_type": "MANAGED" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "storage_location": { + "action": "skip", + "reason": "immutable_backend_value", + "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" + } + } + } } === Deploy renames the catalog and updates the schema and the volume at their new paths diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script index fee34a9d751..4bb99866eef 100644 --- a/acceptance/bundle/resources/catalogs/rename/script +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -21,7 +21,8 @@ title "Rename the catalog" trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ title "UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them" -trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"].changes.catalog_name' +# Drop the remote_state fields that the test server omits or fills differently from UC. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type title "Deploy renames the catalog and updates the schema and the volume at their new paths" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/rename/output.txt b/acceptance/bundle/resources/schemas/rename/output.txt index 253928c8d6c..a2c446f8133 100644 --- a/acceptance/bundle/resources/schemas/rename/output.txt +++ b/acceptance/bundle/resources/schemas/rename/output.txt @@ -23,11 +23,46 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === The plan renames the schema in place instead of recreating it >>> [CLI] bundle plan --output json { - "action": "update_id", - "reason": "id_changes", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-[UNIQUE_NAME]" + "resources.schemas.schema1": { + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "comment": "COMMENT1", + "name": "test-schema-renamed-[UNIQUE_NAME]" + } + }, + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "comment": "COMMENT1", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "test-schema-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "updated_by": "[USERNAME]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + } + } + } } === Deploy renames the schema without a deletion prompt diff --git a/acceptance/bundle/resources/schemas/rename/script b/acceptance/bundle/resources/schemas/rename/script index ec7a4694578..3a16724e09a 100644 --- a/acceptance/bundle/resources/schemas/rename/script +++ b/acceptance/bundle/resources/schemas/rename/script @@ -20,7 +20,7 @@ title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- title "The plan renames the schema in place instead of recreating it" -trace $CLI bundle plan --output json | jq '.plan["resources.schemas.schema1"].changes.name' +trace $CLI bundle plan --output json | jq .plan title "Deploy renames the schema without a deletion prompt" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt index 0e943e097c3..437e6285edd 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -33,11 +33,84 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it >>> [CLI] bundle plan --output json { - "action": "update_id", - "reason": "parent_renamed", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-[UNIQUE_NAME]" + "resources.schemas.schema1": { + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "test-schema-renamed-[UNIQUE_NAME]" + } + }, + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-[UNIQUE_NAME]", + "name": "test-schema-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "main", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "main.test-schema-[UNIQUE_NAME].myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "test-schema-[UNIQUE_NAME]", + "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", + "volume_id": "[UUID]", + "volume_type": "MANAGED" + }, + "changes": { + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + }, + "storage_location": { + "action": "skip", + "reason": "immutable_backend_value", + "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" + } + } + } } === Deploy renames the schema and updates the volume at its new path diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/script b/acceptance/bundle/resources/schemas/rename_with_volume/script index 383f22a7c29..ab582ad9496 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/script +++ b/acceptance/bundle/resources/schemas/rename_with_volume/script @@ -21,7 +21,8 @@ title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- title "UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it" -trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"].changes.schema_name' +# Drop the remote_state fields that the test server omits or fills differently from UC. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id title "Deploy renames the schema and updates the volume at its new path" trace $CLI bundle deploy From f3a59d26b22c31da66ff3dd0f3f43389d0e7d27f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 11:49:52 +0100 Subject: [PATCH 09/75] direct: don't send an empty update to a child moved by its parent's rename UC moves schemas and volumes with their renamed catalog or schema, so the re-key needs no update. Read the child instead to record its full name as UC spells it. Co-authored-by: Isaac --- .../resources/catalogs/rename/output.txt | 17 +------- .../bundle/resources/catalogs/rename/script | 2 +- .../schemas/rename_with_volume/output.txt | 9 +--- .../schemas/rename_with_volume/script | 2 +- bundle/deployplan/plan.go | 11 +++++ bundle/deployplan/plan_test.go | 41 +++++++++++++++++++ bundle/direct/dresources/schema.go | 12 +++++- bundle/direct/dresources/volume.go | 12 +++++- 8 files changed, 78 insertions(+), 28 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index a0d4de8b2d1..71101a2c013 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -149,7 +149,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } } -=== Deploy renames the catalog and updates the schema and the volume at their new paths +=== Deploy renames the catalog. UC moves the schema and the volume with it, so they get no request >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated catalogs.catalog1 @@ -168,21 +168,6 @@ Resources: 0 created, 3 changed, 0 deleted, 0 unchanged "new_name": "test_catalog_renamed_[UNIQUE_NAME]" } } -{ - "method": "PATCH", - "path": "/api/2.1/unity-catalog/schemas/test_catalog_renamed_[UNIQUE_NAME].myschema", - "body": { - "comment": "", - "custom_max_retention_hours": 0 - } -} -{ - "method": "PATCH", - "path": "/api/2.1/unity-catalog/volumes/test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", - "body": { - "comment": "" - } -} === The old catalog is gone and the schema and the volume are under the new one old catalog is gone diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script index 4bb99866eef..1a20a59c050 100644 --- a/acceptance/bundle/resources/catalogs/rename/script +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -24,7 +24,7 @@ title "UC moves the schema and the volume with the catalog, so the plan re-keys # Drop the remote_state fields that the test server omits or fills differently from UC. trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type -title "Deploy renames the catalog and updates the schema and the volume at their new paths" +title "Deploy renames the catalog. UC moves the schema and the volume with it, so they get no request" trace $CLI bundle deploy trace print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt index 437e6285edd..d92d0cfbb20 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -113,7 +113,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } -=== Deploy renames the schema and updates the volume at its new path +=== Deploy renames the schema. UC moves the volume with it, so the volume gets no request >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated schemas.schema1 @@ -131,13 +131,6 @@ Resources: 0 created, 2 changed, 0 deleted, 0 unchanged "new_name": "test-schema-renamed-[UNIQUE_NAME]" } } -{ - "method": "PATCH", - "path": "/api/2.1/unity-catalog/volumes/main.test-schema-renamed-[UNIQUE_NAME].myvolume", - "body": { - "comment": "" - } -} === The old schema is gone and the volume is under the new one >>> musterr [CLI] schemas get main.test-schema-[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/script b/acceptance/bundle/resources/schemas/rename_with_volume/script index ab582ad9496..3837a1b3b2f 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/script +++ b/acceptance/bundle/resources/schemas/rename_with_volume/script @@ -24,7 +24,7 @@ title "UC moves the volume with the schema, so the plan re-keys the volume inste # Drop the remote_state fields that the test server omits or fills differently from UC. trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id -title "Deploy renames the schema and updates the volume at its new path" +title "Deploy renames the schema. UC moves the volume with it, so the volume gets no request" trace $CLI bundle deploy trace print_requests.py //unity-catalog diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index d53de5eee48..f8055e51dd1 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -235,6 +235,17 @@ func (c *Changes) HasChangeExcept(prefixes ...string) bool { return false } +// OnlyParentRenamed reports whether every actionable change follows a parent's rename +// (ReasonParentRenamed). The backend has already moved such a resource, so only its ID changes. +func (c *Changes) OnlyParentRenamed() bool { + for _, change := range *c { + if change.Action != Skip && change.Reason != ReasonParentRenamed { + return false + } + } + return true +} + func (p *Plan) GetActions() []Action { actions := make([]Action, 0, len(p.Plan)) for key, entry := range p.Plan { diff --git a/bundle/deployplan/plan_test.go b/bundle/deployplan/plan_test.go index cee00f42c44..dad1020fcfb 100644 --- a/bundle/deployplan/plan_test.go +++ b/bundle/deployplan/plan_test.go @@ -65,3 +65,44 @@ func TestHasChange(t *testing.T) { }) } } + +func TestOnlyParentRenamed(t *testing.T) { + parentRenamed := &deployplan.ChangeDesc{Action: deployplan.UpdateWithID, Reason: deployplan.ReasonParentRenamed} + tests := []struct { + name string + changes deployplan.Changes + want bool + }{ + { + name: "parent renamed", + changes: deployplan.Changes{"schema_name": parentRenamed}, + want: true, + }, + { + name: "skip change is ignored", + changes: deployplan.Changes{ + "schema_name": parentRenamed, + "storage_location": {Action: deployplan.Skip, Reason: "immutable_backend_value"}, + }, + want: true, + }, + { + name: "own field also changes", + changes: deployplan.Changes{ + "schema_name": parentRenamed, + "comment": {Action: deployplan.Update}, + }, + want: false, + }, + { + name: "own rename", + changes: deployplan.Changes{"name": {Action: deployplan.UpdateWithID, Reason: "id_changes"}}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, tt.changes.OnlyParentRenamed()) + }) + } +} diff --git a/bundle/direct/dresources/schema.go b/bundle/direct/dresources/schema.go index 9b7bdd43b51..5cc53580265 100644 --- a/bundle/direct/dresources/schema.go +++ b/bundle/direct/dresources/schema.go @@ -92,7 +92,7 @@ func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *catalo } // DoUpdateWithID renames the schema in place when the name changes; the ID is the full name, so it changes too. -func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateSchema, _ *PlanEntry) (string, *catalog.SchemaInfo, error) { +func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateSchema, entry *PlanEntry) (string, *catalog.SchemaInfo, error) { nameFromID, err := getNameFromID(id) if err != nil { return "", nil, err @@ -107,6 +107,16 @@ func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config * // stale (see followParentRenames). fullName := config.CatalogName + "." + nameFromID + if entry.Changes.OnlyParentRenamed() { + // Nothing to update. Read the schema to record its full name as UC spells it: UC + // lowercases names, config may not. + response, err := r.DoRead(ctx, fullName) + if err != nil { + return "", nil, err + } + return response.FullName, response, nil + } + response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(fullName, newName, config)) if err != nil || response == nil { return "", nil, err diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 168de23f1c1..0f51f668842 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -83,7 +83,7 @@ func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *catalo return response, err } -func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateVolumeRequestContent, _ *PlanEntry) (string, *catalog.VolumeInfo, error) { +func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateVolumeRequestContent, entry *PlanEntry) (string, *catalog.VolumeInfo, error) { updateRequest := catalog.UpdateVolumeRequestContent{ Comment: config.Comment, Name: "", // Initialized below @@ -104,6 +104,16 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * // leaving id stale (see followParentRenames). updateRequest.Name = config.CatalogName + "." + config.SchemaName + "." + nameFromID + if entry.Changes.OnlyParentRenamed() { + // Nothing to update. Read the volume to record its full name as UC spells it: UC + // lowercases names, config may not. + response, err := r.DoRead(ctx, updateRequest.Name) + if err != nil { + return "", nil, err + } + return response.FullName, response, nil + } + if config.Name != nameFromID { updateRequest.NewName = config.Name } From ad2b996e2f25cd50ab669b524b86011411ba44cf Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 11:53:01 +0100 Subject: [PATCH 10/75] acc: update, rename and recreate a volume after its schema is renamed Co-authored-by: Isaac --- .../rename_then_change/databricks.yml.tmpl | 17 ++ .../schemas/rename_then_change/out.test.toml | 2 + .../schemas/rename_then_change/output.txt | 276 ++++++++++++++++++ .../schemas/rename_then_change/script | 50 ++++ .../schemas/rename_then_change/test.toml | 7 + 5 files changed, 352 insertions(+) create mode 100644 acceptance/bundle/resources/schemas/rename_then_change/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/rename_then_change/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_then_change/output.txt create mode 100644 acceptance/bundle/resources/schemas/rename_then_change/script create mode 100644 acceptance/bundle/resources/schemas/rename_then_change/test.toml diff --git a/acceptance/bundle/resources/schemas/rename_then_change/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_then_change/databricks.yml.tmpl new file mode 100644 index 00000000000..663acec3be0 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_then_change/databricks.yml.tmpl @@ -0,0 +1,17 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema1-$UNIQUE_NAME + schema2: + catalog_name: main + name: test-schema2-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume + comment: c1 diff --git a/acceptance/bundle/resources/schemas/rename_then_change/out.test.toml b/acceptance/bundle/resources/schemas/rename_then_change/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_then_change/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_then_change/output.txt b/acceptance/bundle/resources/schemas/rename_then_change/output.txt new file mode 100644 index 00000000000..625c432a739 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_then_change/output.txt @@ -0,0 +1,276 @@ + +=== Deploy two schemas and a volume in the first one +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created schemas.schema2 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "test-schema1-[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "test-schema2-[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "comment": "c1", + "name": "myvolume", + "schema_name": "test-schema1-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +=== Rename the volume's schema: the volume is re-keyed without a request +>>> update_file.py databricks.yml test-schema1- test-schema1-renamed- + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema1-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema1-renamed-[UNIQUE_NAME]" + } +} + +=== After the rename, update, update_id and recreate of the volume all reach it at its new path +>>> update_file.py databricks.yml comment: c1 comment: c2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema1-renamed-[UNIQUE_NAME].myvolume", + "body": { + "comment": "c2" + } +} + +>>> update_file.py databricks.yml name: myvolume name: myvolume2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema1-renamed-[UNIQUE_NAME].myvolume", + "body": { + "comment": "c2", + "new_name": "myvolume2" + } +} + +>>> update_file.py databricks.yml schemas.schema1.name schemas.schema2.name + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + recreate resources.volumes.volume1 +Recreated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema1-renamed-[UNIQUE_NAME].myvolume2" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "comment": "c2", + "name": "myvolume2", + "schema_name": "test-schema2-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +=== Rename the volume's new schema: the volume is re-keyed again +>>> update_file.py databricks.yml test-schema2- test-schema2-renamed- + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema2 +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema2-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema2-renamed-[UNIQUE_NAME]" + } +} + +=== Update, update_id and recreate work again, moving the volume back to the first schema +>>> update_file.py databricks.yml comment: c2 comment: c3 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema2-renamed-[UNIQUE_NAME].myvolume2", + "body": { + "comment": "c3" + } +} + +>>> update_file.py databricks.yml name: myvolume2 name: myvolume + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema2-renamed-[UNIQUE_NAME].myvolume2", + "body": { + "comment": "c3", + "new_name": "myvolume" + } +} + +>>> update_file.py databricks.yml schemas.schema2.name schemas.schema1.name + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + recreate resources.volumes.volume1 +Recreated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema2-renamed-[UNIQUE_NAME].myvolume" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "comment": "c3", + "name": "myvolume", + "schema_name": "test-schema1-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +=== Rename the first schema back: the volume is re-keyed to its original path +>>> update_file.py databricks.yml test-schema1-renamed- test-schema1- + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema1-renamed-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema1-[UNIQUE_NAME]" + } +} + +=== The state tracks the current paths and the plan is a no-op +>>> [CLI] bundle summary --output json +{ + "schemas": { + "schema1": "main.test-schema1-[UNIQUE_NAME]", + "schema2": "main.test-schema2-renamed-[UNIQUE_NAME]" + }, + "volumes": { + "volume1": "main.test-schema1-[UNIQUE_NAME].myvolume" + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] volumes read main.test-schema1-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema1-[UNIQUE_NAME].myvolume", + "comment": "c3" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.schemas.schema2 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + delete resources.schemas.schema2 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/schemas/rename_then_change/script b/acceptance/bundle/resources/schemas/rename_then_change/script new file mode 100644 index 00000000000..d89bfb882f8 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_then_change/script @@ -0,0 +1,50 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +deploy() { + trace $CLI bundle deploy "$@" + trace print_requests.py //unity-catalog +} + +title "Deploy two schemas and a volume in the first one" +deploy + +title "Rename the volume's schema: the volume is re-keyed without a request" +trace update_file.py databricks.yml test-schema1- test-schema1-renamed- +deploy + +title "After the rename, update, update_id and recreate of the volume all reach it at its new path" +trace update_file.py databricks.yml "comment: c1" "comment: c2" +deploy +trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" +deploy +trace update_file.py databricks.yml schemas.schema1.name schemas.schema2.name +deploy --auto-approve + +title "Rename the volume's new schema: the volume is re-keyed again" +trace update_file.py databricks.yml test-schema2- test-schema2-renamed- +deploy + +title "Update, update_id and recreate work again, moving the volume back to the first schema" +trace update_file.py databricks.yml "comment: c2" "comment: c3" +deploy +trace update_file.py databricks.yml "name: myvolume2" "name: myvolume" +deploy +trace update_file.py databricks.yml schemas.schema2.name schemas.schema1.name +deploy --auto-approve + +title "Rename the first schema back: the volume is re-keyed to its original path" +trace update_file.py databricks.yml test-schema1-renamed- test-schema1- +deploy + +title "The state tracks the current paths and the plan is a no-op" +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' +trace $CLI bundle plan +trace $CLI volumes read "main.test-schema1-${UNIQUE_NAME}.myvolume" | jq '{full_name, comment}' diff --git a/acceptance/bundle/resources/schemas/rename_then_change/test.toml b/acceptance/bundle/resources/schemas/rename_then_change/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_then_change/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] From 6ffe8a9218eeb7d63b331f35656dceb07250b8ae Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 11:57:50 +0100 Subject: [PATCH 11/75] acc: sort requests of the parallel schema creates Co-authored-by: Isaac --- .../bundle/resources/schemas/rename_then_change/output.txt | 2 +- acceptance/bundle/resources/schemas/rename_then_change/script | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resources/schemas/rename_then_change/output.txt b/acceptance/bundle/resources/schemas/rename_then_change/output.txt index 625c432a739..ed430978177 100644 --- a/acceptance/bundle/resources/schemas/rename_then_change/output.txt +++ b/acceptance/bundle/resources/schemas/rename_then_change/output.txt @@ -8,7 +8,7 @@ Created volumes.volume1 Files: 2 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //unity-catalog +>>> print_requests.py --sort //unity-catalog { "method": "POST", "path": "/api/2.1/unity-catalog/schemas", diff --git a/acceptance/bundle/resources/schemas/rename_then_change/script b/acceptance/bundle/resources/schemas/rename_then_change/script index d89bfb882f8..452af92148c 100644 --- a/acceptance/bundle/resources/schemas/rename_then_change/script +++ b/acceptance/bundle/resources/schemas/rename_then_change/script @@ -14,7 +14,9 @@ deploy() { } title "Deploy two schemas and a volume in the first one" -deploy +trace $CLI bundle deploy +# The two schemas are created in parallel. +trace print_requests.py --sort //unity-catalog title "Rename the volume's schema: the volume is re-keyed without a request" trace update_file.py databricks.yml test-schema1- test-schema1-renamed- From 4c9238fb5103367f20d179279754529eb3f0058b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 12:07:31 +0100 Subject: [PATCH 12/75] acc: a job reading a volume's full_name and id follows its schema's rename Co-authored-by: Isaac --- .../databricks.yml.tmpl | 21 +++++ .../rename_with_volume_ref/out.test.toml | 2 + .../schemas/rename_with_volume_ref/output.txt | 87 +++++++++++++++++++ .../schemas/rename_with_volume_ref/script | 30 +++++++ .../schemas/rename_with_volume_ref/test.toml | 7 ++ 5 files changed, 147 insertions(+) create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume_ref/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume_ref/script create mode 100644 acceptance/bundle/resources/schemas/rename_with_volume_ref/test.toml diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl new file mode 100644 index 00000000000..2624f14c7cf --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl @@ -0,0 +1,21 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume + jobs: + job1: + name: test-job-$UNIQUE_NAME + parameters: + - name: volume_full_name + default: ${resources.volumes.volume1.full_name} + - name: volume_id + default: ${resources.volumes.volume1.id} diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/out.test.toml b/acceptance/bundle/resources/schemas/rename_with_volume_ref/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt new file mode 100644 index 00000000000..47697e156fd --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt @@ -0,0 +1,87 @@ + +=== Deploy a schema, a volume in it and a job that reads the volume's full_name and id +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created jobs.job1 +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] jobs get [JOB_ID] +[ + { + "default": "main.test-schema-[UNIQUE_NAME].myvolume", + "name": "volume_full_name" + }, + { + "default": "main.test-schema-[UNIQUE_NAME].myvolume", + "name": "volume_id" + } +] + +=== Rename the schema +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +=== The volume's full_name and id change with the schema, so the job is updated too +>>> [CLI] bundle plan +update jobs.job1 +update schemas.schema1 +update volumes.volume1 + +Plan: 0 to add, 3 to change, 0 to delete, 0 unchanged + +=== Deploy passes the volume's new full_name and id to the job +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated jobs.job1 +Updated schemas.schema1 +Updated volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 3 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} + +>>> [CLI] jobs get [JOB_ID] +[ + { + "default": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "name": "volume_full_name" + }, + { + "default": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "name": "volume_id" + } +] + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.job1 + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/script b/acceptance/bundle/resources/schemas/rename_with_volume_ref/script new file mode 100644 index 00000000000..0b1627161b3 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/script @@ -0,0 +1,30 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema, a volume in it and a job that reads the volume's full_name and id" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +JOB_ID=$($CLI bundle summary --output json | jq -r .resources.jobs.job1.id) +add_repl "$JOB_ID" JOB_ID +trace $CLI jobs get "$JOB_ID" | jq .settings.parameters + +title "Rename the schema" +trace update_file.py databricks.yml test-schema- test-schema-renamed- + +title "The volume's full_name and id change with the schema, so the job is updated too" +trace $CLI bundle plan + +title "Deploy passes the volume's new full_name and id to the job" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI jobs get "$JOB_ID" | jq .settings.parameters + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/test.toml b/acceptance/bundle/resources/schemas/rename_with_volume_ref/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] From cb1b16f799b2b97dd255c10d5c437d5292dcd8a1 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 12:58:54 +0100 Subject: [PATCH 13/75] acc: drop cloud-specific fields from the rename plans Azure and GCP return their own volume storage_location and predictive optimization metastore, and UC may touch a schema's updated_at after create. Co-authored-by: Isaac --- .../bundle/resources/catalogs/rename/output.txt | 6 ------ acceptance/bundle/resources/catalogs/rename/script | 4 ++-- acceptance/bundle/resources/schemas/rename/output.txt | 11 ++--------- acceptance/bundle/resources/schemas/rename/script | 3 ++- .../resources/schemas/rename_with_volume/output.txt | 11 ----------- .../resources/schemas/rename_with_volume/script | 4 ++-- 6 files changed, 8 insertions(+), 31 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index 71101a2c013..6534dc8620b 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -128,7 +128,6 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "name": "myvolume", "owner": "[USERNAME]", "schema_name": "myschema", - "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", "volume_id": "[UUID]", "volume_type": "MANAGED" }, @@ -139,11 +138,6 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "old": "test_catalog_[UNIQUE_NAME]", "new": "test_catalog_renamed_[UNIQUE_NAME]", "remote": "test_catalog_[UNIQUE_NAME]" - }, - "storage_location": { - "action": "skip", - "reason": "immutable_backend_value", - "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" } } } diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script index 1a20a59c050..50ee123b666 100644 --- a/acceptance/bundle/resources/catalogs/rename/script +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -21,8 +21,8 @@ title "Rename the catalog" trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ title "UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them" -# Drop the remote_state fields that the test server omits or fills differently from UC. -trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location title "Deploy renames the catalog. UC moves the schema and the volume with it, so they get no request" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/rename/output.txt b/acceptance/bundle/resources/schemas/rename/output.txt index a2c446f8133..d14df16f2e1 100644 --- a/acceptance/bundle/resources/schemas/rename/output.txt +++ b/acceptance/bundle/resources/schemas/rename/output.txt @@ -37,21 +37,14 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "catalog_name": "main", "catalog_type": "MANAGED_CATALOG", "comment": "COMMENT1", - "created_at": [UNIX_TIME_MILLIS][0], + "created_at": [UNIX_TIME_MILLIS], "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, "enable_predictive_optimization": "INHERIT", "full_name": "main.test-schema-[UNIQUE_NAME]", "metastore_id": "[UUID]", "name": "test-schema-[UNIQUE_NAME]", "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "updated_by": "[USERNAME]" + "schema_id": "[UUID]" }, "changes": { "name": { diff --git a/acceptance/bundle/resources/schemas/rename/script b/acceptance/bundle/resources/schemas/rename/script index 3a16724e09a..bd155193e9e 100644 --- a/acceptance/bundle/resources/schemas/rename/script +++ b/acceptance/bundle/resources/schemas/rename/script @@ -20,7 +20,8 @@ title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- title "The plan renames the schema in place instead of recreating it" -trace $CLI bundle plan --output json | jq .plan +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by effective_predictive_optimization_flag title "Deploy renames the schema without a deletion prompt" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt index d92d0cfbb20..260614b1f8a 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -47,11 +47,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "catalog_type": "MANAGED_CATALOG", "created_at": [UNIX_TIME_MILLIS][0], "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, "enable_predictive_optimization": "INHERIT", "full_name": "main.test-schema-[UNIQUE_NAME]", "name": "test-schema-[UNIQUE_NAME]", @@ -92,7 +87,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "name": "myvolume", "owner": "[USERNAME]", "schema_name": "test-schema-[UNIQUE_NAME]", - "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", "volume_id": "[UUID]", "volume_type": "MANAGED" }, @@ -103,11 +97,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "old": "test-schema-[UNIQUE_NAME]", "new": "test-schema-renamed-[UNIQUE_NAME]", "remote": "test-schema-[UNIQUE_NAME]" - }, - "storage_location": { - "action": "skip", - "reason": "immutable_backend_value", - "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" } } } diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/script b/acceptance/bundle/resources/schemas/rename_with_volume/script index 3837a1b3b2f..f3591d9f38b 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/script +++ b/acceptance/bundle/resources/schemas/rename_with_volume/script @@ -21,8 +21,8 @@ title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- title "UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it" -# Drop the remote_state fields that the test server omits or fills differently from UC. -trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location title "Deploy renames the schema. UC moves the volume with it, so the volume gets no request" trace $CLI bundle deploy From 4bb34179cb3f54359b3a32602349303ab8b2657d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 14:09:32 +0100 Subject: [PATCH 14/75] acc: show preserved schema and volume ids through replacements Co-authored-by: Isaac --- .../bundle/resources/catalogs/rename/output.txt | 16 +++++++--------- .../bundle/resources/catalogs/rename/script | 14 +++++--------- .../bundle/resources/schemas/rename/output.txt | 10 ++++------ .../bundle/resources/schemas/rename/script | 13 +++---------- .../schemas/rename_with_volume/output.txt | 10 ++++------ .../resources/schemas/rename_with_volume/script | 13 +++---------- 6 files changed, 26 insertions(+), 50 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index 6534dc8620b..07843301bcc 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -88,7 +88,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "full_name": "test_catalog_[UNIQUE_NAME].myschema", "name": "myschema", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[SCHEMA_ID]" }, "changes": { "catalog_name": { @@ -128,7 +128,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "name": "myvolume", "owner": "[USERNAME]", "schema_name": "myschema", - "volume_id": "[UUID]", + "volume_id": "[VOLUME_ID]", "volume_type": "MANAGED" }, "changes": { @@ -163,25 +163,23 @@ Resources: 0 created, 3 changed, 0 deleted, 0 unchanged } } -=== The old catalog is gone and the schema and the volume are under the new one +=== The old catalog is gone and the schema and the volume are under the new one, with their original ids old catalog is gone >>> [CLI] schemas get test_catalog_renamed_[UNIQUE_NAME].myschema { "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema", - "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]" + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" } >>> [CLI] volumes read test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume { "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", - "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]" + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "volume_id": "[VOLUME_ID]" } -=== The schema_id and volume_id are preserved, so these are the same objects -schema_id is unchanged -volume_id is unchanged - === The state tracks the new paths >>> [CLI] bundle summary --output json { diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script index 50ee123b666..e84ef070fc8 100644 --- a/acceptance/bundle/resources/catalogs/rename/script +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -14,8 +14,8 @@ trap cleanup EXIT title "Deploy a catalog with a schema and a volume in it" trace $CLI bundle deploy trace print_requests.py //unity-catalog -SCHEMA_ID=$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id) -VOLUME_ID=$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id) +add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID title "Rename the catalog" trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ @@ -28,15 +28,11 @@ title "Deploy renames the catalog. UC moves the schema and the volume with it, s trace $CLI bundle deploy trace print_requests.py //unity-catalog -title "The old catalog is gone and the schema and the volume are under the new one\n" +title "The old catalog is gone and the schema and the volume are under the new one, with their original ids\n" # Not traced: the "not found" message differs between UC and the test server. $CLI catalogs get "$OLD_CATALOG" &> /dev/null && echo "old catalog still exists" || echo "old catalog is gone" -trace $CLI schemas get "$NEW_CATALOG.myschema" | jq '{full_name, catalog_name}' -trace $CLI volumes read "$NEW_CATALOG.myschema.myvolume" | jq '{full_name, catalog_name}' - -title "The schema_id and volume_id are preserved, so these are the same objects\n" -[ "$($CLI schemas get "$NEW_CATALOG.myschema" | jq -r .schema_id)" = "$SCHEMA_ID" ] && echo "schema_id is unchanged" || echo "schema_id changed" -[ "$($CLI volumes read "$NEW_CATALOG.myschema.myvolume" | jq -r .volume_id)" = "$VOLUME_ID" ] && echo "volume_id is unchanged" || echo "volume_id changed" +trace $CLI schemas get "$NEW_CATALOG.myschema" | jq '{full_name, catalog_name, schema_id}' +trace $CLI volumes read "$NEW_CATALOG.myschema.myvolume" | jq '{full_name, catalog_name, volume_id}' title "The state tracks the new paths" trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' diff --git a/acceptance/bundle/resources/schemas/rename/output.txt b/acceptance/bundle/resources/schemas/rename/output.txt index d14df16f2e1..275d84698ac 100644 --- a/acceptance/bundle/resources/schemas/rename/output.txt +++ b/acceptance/bundle/resources/schemas/rename/output.txt @@ -44,7 +44,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "metastore_id": "[UUID]", "name": "test-schema-[UNIQUE_NAME]", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[SCHEMA_ID]" }, "changes": { "name": { @@ -76,7 +76,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } } -=== The schema is only reachable under the new name +=== The schema is only reachable under the new name, with its original schema_id >>> musterr [CLI] schemas get main.test-schema-[UNIQUE_NAME] Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. @@ -84,12 +84,10 @@ Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. { "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", "name": "test-schema-renamed-[UNIQUE_NAME]", - "comment": "COMMENT1" + "comment": "COMMENT1", + "schema_id": "[SCHEMA_ID]" } -=== The schema_id is preserved, so this is the same schema -schema_id is unchanged - === The state and summary track the new name >>> [CLI] bundle summary --output json { diff --git a/acceptance/bundle/resources/schemas/rename/script b/acceptance/bundle/resources/schemas/rename/script index bd155193e9e..c8617676747 100644 --- a/acceptance/bundle/resources/schemas/rename/script +++ b/acceptance/bundle/resources/schemas/rename/script @@ -14,7 +14,7 @@ trap cleanup EXIT title "Deploy the schema" trace $CLI bundle deploy trace print_requests.py //unity-catalog -SCHEMA_ID=$($CLI schemas get "$OLD_SCHEMA" | jq -r .schema_id) +add_repl "$($CLI schemas get "$OLD_SCHEMA" | jq -r .schema_id)" SCHEMA_ID title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- @@ -27,16 +27,9 @@ title "Deploy renames the schema without a deletion prompt" trace $CLI bundle deploy trace print_requests.py //unity-catalog -title "The schema is only reachable under the new name" +title "The schema is only reachable under the new name, with its original schema_id" trace musterr $CLI schemas get "$OLD_SCHEMA" -trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, name, comment}' - -title "The schema_id is preserved, so this is the same schema\n" -if [ "$($CLI schemas get "$NEW_SCHEMA" | jq -r .schema_id)" = "$SCHEMA_ID" ]; then - echo "schema_id is unchanged" -else - echo "schema_id changed" -fi +trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, name, comment, schema_id}' title "The state and summary track the new name" trace $CLI bundle summary --output json | jq '.resources.schemas.schema1 | {id, name}' diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt index 260614b1f8a..37458a8f7bf 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -87,7 +87,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "name": "myvolume", "owner": "[USERNAME]", "schema_name": "test-schema-[UNIQUE_NAME]", - "volume_id": "[UUID]", + "volume_id": "[VOLUME_ID]", "volume_type": "MANAGED" }, "changes": { @@ -121,19 +121,17 @@ Resources: 0 created, 2 changed, 0 deleted, 0 unchanged } } -=== The old schema is gone and the volume is under the new one +=== The old schema is gone and the volume is under the new one, with its original volume_id >>> musterr [CLI] schemas get main.test-schema-[UNIQUE_NAME] Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. >>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume { "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", - "schema_name": "test-schema-renamed-[UNIQUE_NAME]" + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_id": "[VOLUME_ID]" } -=== The volume_id is preserved, so this is the same volume -volume_id is unchanged - === The state tracks the new path >>> [CLI] bundle summary --output json "main.test-schema-renamed-[UNIQUE_NAME].myvolume" diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/script b/acceptance/bundle/resources/schemas/rename_with_volume/script index f3591d9f38b..fd5830bf18a 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/script +++ b/acceptance/bundle/resources/schemas/rename_with_volume/script @@ -15,7 +15,7 @@ trap cleanup EXIT title "Deploy a schema with a volume in it" trace $CLI bundle deploy trace print_requests.py //unity-catalog -VOLUME_ID=$($CLI volumes read "$OLD_VOLUME" | jq -r .volume_id) +add_repl "$($CLI volumes read "$OLD_VOLUME" | jq -r .volume_id)" VOLUME_ID title "Rename the schema" trace update_file.py databricks.yml test-schema- test-schema-renamed- @@ -28,16 +28,9 @@ title "Deploy renames the schema. UC moves the volume with it, so the volume get trace $CLI bundle deploy trace print_requests.py //unity-catalog -title "The old schema is gone and the volume is under the new one" +title "The old schema is gone and the volume is under the new one, with its original volume_id" trace musterr $CLI schemas get "$OLD_SCHEMA" -trace $CLI volumes read "$NEW_VOLUME" | jq '{full_name, schema_name}' - -title "The volume_id is preserved, so this is the same volume\n" -if [ "$($CLI volumes read "$NEW_VOLUME" | jq -r .volume_id)" = "$VOLUME_ID" ]; then - echo "volume_id is unchanged" -else - echo "volume_id changed" -fi +trace $CLI volumes read "$NEW_VOLUME" | jq '{full_name, schema_name, volume_id}' title "The state tracks the new path" trace $CLI bundle summary --output json | jq '.resources.volumes.volume1.id' From 688fda987cb412de72fa234eecbbf2a32a021cd6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 14:26:20 +0100 Subject: [PATCH 15/75] direct: build the re-keyed child's ID from config instead of reading it Like the other DoUpdateWithID implementations, take the new ID from config, so a child moved by its parent's rename needs no request at all. Co-authored-by: Isaac --- bundle/direct/dresources/schema.go | 9 ++------- bundle/direct/dresources/volume.go | 9 ++------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/bundle/direct/dresources/schema.go b/bundle/direct/dresources/schema.go index 5cc53580265..8a928cb8989 100644 --- a/bundle/direct/dresources/schema.go +++ b/bundle/direct/dresources/schema.go @@ -108,13 +108,8 @@ func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config * fullName := config.CatalogName + "." + nameFromID if entry.Changes.OnlyParentRenamed() { - // Nothing to update. Read the schema to record its full name as UC spells it: UC - // lowercases names, config may not. - response, err := r.DoRead(ctx, fullName) - if err != nil { - return "", nil, err - } - return response.FullName, response, nil + // UC already moved the schema with its catalog; only the ID changes. + return fullName, nil, nil } response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(fullName, newName, config)) diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 0f51f668842..445cd826314 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -105,13 +105,8 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * updateRequest.Name = config.CatalogName + "." + config.SchemaName + "." + nameFromID if entry.Changes.OnlyParentRenamed() { - // Nothing to update. Read the volume to record its full name as UC spells it: UC - // lowercases names, config may not. - response, err := r.DoRead(ctx, updateRequest.Name) - if err != nil { - return "", nil, err - } - return response.FullName, response, nil + // UC already moved the volume with its parent; only the ID changes. + return updateRequest.Name, nil, nil } if config.Name != nameFromID { From a9b667f09a1fa4410c21196994a75f00a41eb8be Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 16:26:08 +0100 Subject: [PATCH 16/75] direct: follow a parent rename that an interrupted deploy already applied A deploy can stop after renaming a schema but before re-keying its volume. The next plan then saw no rename in progress, read the volume at its old path, got 404, and planned a create, which fails on UC with RESOURCE_ALREADY_EXISTS. An update_id now records the old values of the fields it changes (renamed_from) in the same state write as the new ID. A child whose parent is not being renamed in the plan follows that recorded rename instead, and is not planned as a create just because its read at the old ID found nothing. Co-authored-by: Isaac --- .../rename_interrupted/databricks.yml.tmpl | 14 ++ .../schemas/rename_interrupted/out.test.toml | 2 + .../schemas/rename_interrupted/output.txt | 157 ++++++++++++++++++ .../schemas/rename_interrupted/script | 38 +++++ .../schemas/rename_interrupted/test.toml | 29 ++++ bundle/direct/apply.go | 28 +++- bundle/direct/bundle_plan.go | 41 +++-- bundle/direct/bundle_plan_test.go | 4 +- bundle/direct/dstate/dms.go | 12 +- bundle/direct/dstate/dms_test.go | 13 ++ bundle/direct/dstate/state.go | 25 ++- bundle/direct/dstate/state_test.go | 17 ++ 12 files changed, 351 insertions(+), 29 deletions(-) create mode 100644 acceptance/bundle/resources/schemas/rename_interrupted/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/rename_interrupted/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_interrupted/output.txt create mode 100644 acceptance/bundle/resources/schemas/rename_interrupted/script create mode 100644 acceptance/bundle/resources/schemas/rename_interrupted/test.toml diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_interrupted/databricks.yml.tmpl new file mode 100644 index 00000000000..2964856d08e --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_interrupted/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume + comment: c1 diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/out.test.toml b/acceptance/bundle/resources/schemas/rename_interrupted/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_interrupted/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt new file mode 100644 index 00000000000..091a96903cb --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt @@ -0,0 +1,157 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema and change the volume's comment. The CLI is killed on the volume's update, after the schema's rename is saved +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> update_file.py databricks.yml comment: c1 comment: c2 + +>>> errcode [CLI] bundle deploy -q +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +[PROCESS_KILLED] + +Exit code: [KILLED] + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} + +=== The next plan still re-keys the volume, whose state has the old schema +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "comment": "", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "custom_max_retention_hours": 0, + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "test-schema-renamed-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][1], + "updated_by": "[USERNAME]" + }, + "changes": { + "comment": { + "action": "skip", + "reason": "empty", + "remote": "" + }, + "custom_max_retention_hours": { + "action": "skip", + "reason": "empty", + "remote": 0 + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "comment": "c2", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "changes": { + "comment": { + "action": "update", + "old": "c1", + "new": "c2" + }, + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]" + } + } + } +} + +=== It can, because the schema's old name was saved together with its rename +>>> print_state.py +{ + "__id__": "main.test-schema-renamed-[UNIQUE_NAME]", + "renamed_from": { + "name": "test-schema-[UNIQUE_NAME]" + } +} + +=== The next deploy updates the volume at its new path +>>> [CLI] bundle deploy --force-lock +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "body": { + "comment": "c2" + } +} + +>>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "comment": "c2", + "volume_id": "[VOLUME_ID]" +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve --force-lock +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/script b/acceptance/bundle/resources/schemas/rename_interrupted/script new file mode 100644 index 00000000000..ca0d499bb7e --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_interrupted/script @@ -0,0 +1,38 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +NEW_VOLUME="main.test-schema-renamed-${UNIQUE_NAME}.myvolume" + +cleanup() { + trace $CLI bundle destroy --auto-approve --force-lock + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI volumes read "main.test-schema-${UNIQUE_NAME}.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the schema and change the volume's comment. The CLI is killed on the volume's update, after the schema's rename is saved" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +trace update_file.py databricks.yml "comment: c1" "comment: c2" +kill_after.py "PATCH /api/2.1/unity-catalog/volumes/$NEW_VOLUME" 0 1 +# -q: the per-resource lines race the kill signal. +trace errcode $CLI bundle deploy -q +trace print_requests.py //unity-catalog + +title "The next plan still re-keys the volume, whose state has the old schema" +trace $CLI bundle plan --output json | jq .plan + +title "It can, because the schema's old name was saved together with its rename" +trace print_state.py | jq '.state["resources.schemas.schema1"] | {__id__, renamed_from}' + +title "The next deploy updates the volume at its new path" +trace $CLI bundle deploy --force-lock +trace print_requests.py //unity-catalog +trace $CLI volumes read "$NEW_VOLUME" | jq '{full_name, comment, volume_id}' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/test.toml b/acceptance/bundle/resources/schemas/rename_interrupted/test.toml new file mode 100644 index 00000000000..284671b7e15 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_interrupted/test.toml @@ -0,0 +1,29 @@ +# Local-only: the test server kills the CLI mid-deploy (kill_after.py), which a real workspace cannot. +Cloud = false + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] + +# KillCaller needs the CLI's pid. A kill leaves no state in DMS to recover from, so DMS is not run. +Env.DATABRICKS_CLI_TEST_PID = "1" +EnvMatrix.DMS = [""] + +# Normalize how a killed process shows up on each OS, as in bundle/deploy/wal. +[[Repls]] +Old = 'script: line \d+:\s+\d+ Killed(: 9)?\s+"\$@"' +New = '[PROCESS_KILLED]' + +[[Repls]] +Old = '(\n>>> errcode [^\n]+\n)\nExit code:' +New = "${1}[PROCESS_KILLED]\n\nExit code:" + +[[Repls]] +Old = 'Exit code: 137' +New = 'Exit code: [KILLED]' + +[[Repls]] +Old = '(\[PROCESS_KILLED\]\n\nExit code: )1' +New = '${1}[KILLED]' diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index dc0751f03b3..2e7bb91b656 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -89,7 +89,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn) + err = d.saveState(ctx, db, newID, newState, d.DependsOn, nil) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -187,7 +187,7 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("deleting state id=%s: %w", id, err) } } else { - err = d.saveState(ctx, db, id, newState, d.DependsOn) + err = d.saveState(ctx, db, id, newState, d.DependsOn, nil) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -236,7 +236,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn) + err = d.saveState(ctx, db, newID, newState, d.DependsOn, renamedFrom(planEntry.Changes)) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -329,7 +329,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = d.saveState(ctx, db, id, newState, d.DependsOn) + err = d.saveState(ctx, db, id, newState, d.DependsOn, nil) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -341,7 +341,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, // with content hashes, see dresources.CompactState) before persisting it. Fields already // smaller than a hash placeholder are persisted as is. Sensitive fields are replaced by a // placeholder value so secrets are never written to disk in plaintext. -func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { +func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry, renamedFrom map[string]any) error { compacted, err := dresources.CompactState(d.Adapter.ResourceConfig(), state) if err != nil { return fmt.Errorf("compacting state: %w", err) @@ -349,7 +349,23 @@ func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentSta if err := zeroSensitiveFields(d.Adapter, compacted); err != nil { return fmt.Errorf("redacting state: %w", err) } - return db.SaveState(ctx, d.ResourceKey, newID, compacted, dependsOn) + return db.SaveStateRenamed(ctx, d.ResourceKey, newID, compacted, dependsOn, renamedFrom) +} + +// renamedFrom returns the old values of the fields an update_id changes, for the state entry's +// RenamedFrom. +func renamedFrom(changes deployplan.Changes) map[string]any { + var result map[string]any + for path, change := range changes { + if change.Action != deployplan.UpdateWithID { + continue + } + if result == nil { + result = make(map[string]any) + } + result[path] = change.Old + } + return result } func parseState(destType reflect.Type, raw json.RawMessage) (any, error) { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 45bf1bdcce9..1eb89c6499f 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -377,15 +377,16 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } - err = followParentRenames(adapter, entry.Changes, parentRenames) + followedRename, err := followParentRenames(adapter, entry.Changes, parentRenames) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: classifying changes: %w", errorPrefix, err)) return false } - if remoteState == nil { + if remoteState == nil && !followedRename { // Even if local action is "recreate" which is higher than "create", we should still pick "create" here - // because we know remote does not exist. + // because we know remote does not exist. A child that followed its parent's rename is not + // missing: it moved, and the read used its old ID. action = deployplan.Create } else { action = getMaxAction(entry.Changes) @@ -711,6 +712,9 @@ func (b *DeploymentBundle) parentRenames(resourceKey string) (map[string]*deploy } // renameChange returns the change to field of resourceKey if it is part of an update_id, nil otherwise. +// +// When this plan does not rename the field, the rename recorded with the resource's state counts +// too: a deploy interrupted after renaming the parent leaves its children under the old name. func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan.ChangeDesc, error) { entry, err := b.Plan.ReadLockEntry(resourceKey) if err != nil { @@ -719,14 +723,25 @@ func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan. defer b.Plan.ReadUnlockEntry(resourceKey) // A missing entry is reported by resolveReferences. - if entry == nil || entry.Action != deployplan.UpdateWithID { + if entry == nil { + return nil, nil + } + if change := entry.Changes[field]; change != nil && change.Action == deployplan.UpdateWithID { + return change, nil + } + // A parent that is created or recreated is a new object; UC does not move children into it. + if !entry.Action.KeepsID() && entry.Action != deployplan.UpdateWithID { + return nil, nil + } + dbentry, ok := b.StateDB.GetResourceEntry(resourceKey) + if !ok { return nil, nil } - change := entry.Changes[field] - if change == nil || change.Action != deployplan.UpdateWithID { + old, ok := dbentry.RenamedFrom[field] + if !ok { return nil, nil } - return change, nil + return &deployplan.ChangeDesc{Action: deployplan.UpdateWithID, Old: old}, nil } // followParentRenames plans an update_id instead of a recreate for an ID field that changes @@ -736,10 +751,13 @@ func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan. // // The old values must match too: a child moved to the renamed parent from somewhere else // still has to be recreated. -func followParentRenames(adapter *dresources.Adapter, changes deployplan.Changes, parentRenames map[string]*deployplan.ChangeDesc) error { +// +// It reports whether any change follows a rename. +func followParentRenames(adapter *dresources.Adapter, changes deployplan.Changes, parentRenames map[string]*deployplan.ChangeDesc) (bool, error) { if !adapter.HasDoUpdateWithID() { - return nil + return false, nil } + followed := false for pathString, parentChange := range parentRenames { ch := changes[pathString] if ch == nil || ch.Action != deployplan.Recreate || !structdiff.IsEqual(ch.Old, parentChange.Old) { @@ -747,15 +765,16 @@ func followParentRenames(adapter *dresources.Adapter, changes deployplan.Changes } path, err := structpath.ParsePath(pathString) if err != nil { - return err + return false, err } if _, ok := findMatchingRule(path, adapter.ResourceConfig().ProvidedIDFields); !ok { continue } ch.Action = deployplan.UpdateWithID ch.Reason = deployplan.ReasonParentRenamed + followed = true } - return nil + return followed, nil } // shouldSkipNormalized skips a change that is a false diff caused by UC API diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index f50de986918..7d735755080 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -323,8 +323,10 @@ func TestFollowParentRenames(t *testing.T) { parentRenames := map[string]*deployplan.ChangeDesc{ tt.field: {Action: deployplan.UpdateWithID, Reason: "id_changes", Old: "s_old", New: "s_new", Remote: "s_old"}, } - require.NoError(t, followParentRenames(adapter, deployplan.Changes{tt.field: ch}, parentRenames)) + followed, err := followParentRenames(adapter, deployplan.Changes{tt.field: ch}, parentRenames) + require.NoError(t, err) assert.Equal(t, tt.expectedAction, ch.Action) + assert.Equal(t, tt.expectedAction == deployplan.UpdateWithID, followed) }) } } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 57e10b8d4b4..3f38ddc2bd8 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -12,8 +12,9 @@ import ( // config so depends_on survives the round trip: DMS has no field for dependency edges, and // nesting them in the config would collide with resource fields of the same name. type RecordedState struct { - State json.RawMessage `json:"state"` - DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` + State json.RawMessage `json:"state"` + DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` + RenamedFrom map[string]any `json:"renamed_from,omitempty"` } // applyDMSState fills in the resource state from what DMS recorded. For a deployment that uses DMS @@ -54,8 +55,9 @@ func stateEntry(res dms.Resource) (ResourceEntry, error) { } return ResourceEntry{ - ID: res.ID, - State: recorded.State, - DependsOn: recorded.DependsOn, + ID: res.ID, + State: recorded.State, + DependsOn: recorded.DependsOn, + RenamedFrom: recorded.RenamedFrom, }, nil } diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index ab4e1bc017f..18a1aceddcc 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -44,6 +44,19 @@ func TestApplyDMSState(t *testing.T) { "resources.pipelines.bar": {ID: "456"}, }, }, + { + name: "renamed_from comes back from the envelope", + recorded: []dms.Resource{ + {Key: "resources.schemas.foo", ID: "main.new", State: `{"state":{"name":"new"},"renamed_from":{"name":"old"}}`}, + }, + want: map[string]ResourceEntry{ + "resources.schemas.foo": { + ID: "main.new", + State: json.RawMessage(`{"name":"new"}`), + RenamedFrom: map[string]any{"name": "old"}, + }, + }, + }, { // Nil is what a list holding nothing returns, and it means a successful deploy of // nothing rather than missing data. diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index de00e76233a..ace32a78770 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -169,6 +169,12 @@ type ResourceEntry struct { ID string `json:"__id__"` State json.RawMessage `json:"state"` DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` + + // RenamedFrom holds the old values of the fields an in-place ID change (update_id) changed, + // keyed by field path, until the resource is saved again. It is saved in the same write as + // the new ID, so a child that an interrupted deploy left under the old name can still follow + // the rename. + RenamedFrom map[string]any `json:"renamed_from,omitempty"` } type WALEntry struct { @@ -289,7 +295,7 @@ func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause e var recorded json.RawMessage if entry, ok := db.GetResourceEntry(resourceKey); resourceID != "" && ok && len(entry.State) > 0 { var err error - recorded, err = json.Marshal(RecordedState{State: entry.State, DependsOn: entry.DependsOn}) + recorded, err = json.Marshal(RecordedState{State: entry.State, DependsOn: entry.DependsOn, RenamedFrom: entry.RenamedFrom}) if err != nil { // Nothing the caller can act on, so record the failure without the state. recorded = nil @@ -315,6 +321,12 @@ func NewDatabase(lineage string, serial int) Database { // SaveState records the resource's state after an operation was applied to it. func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { + return db.SaveStateRenamed(ctx, key, newID, state, dependsOn, nil) +} + +// SaveStateRenamed is SaveState for an ID changed in place: renamedFrom (see ResourceEntry) is +// recorded together with the new ID. +func (db *DeploymentState) SaveStateRenamed(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry, renamedFrom map[string]any) error { db.AssertOpenedForWrite() db.mu.Lock() defer db.mu.Unlock() @@ -328,9 +340,10 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta return err } entry := ResourceEntry{ - ID: newID, - State: json.RawMessage(jsonMessage), - DependsOn: dependsOn, + ID: newID, + State: json.RawMessage(jsonMessage), + DependsOn: dependsOn, + RenamedFrom: renamedFrom, } switch db.mode { @@ -344,7 +357,7 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta db.Data.State[key] = entry db.stateIDs[key] = newID if buf := db.operationBuffer; buf != nil { - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, RenamedFrom: renamedFrom}) if err != nil { return err } @@ -359,7 +372,7 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta } db.stateIDs[key] = newID if buf := db.operationBuffer; buf != nil { - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, RenamedFrom: renamedFrom}) if err != nil { return err } diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index c5bfea2783b..9e22d15b8a1 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -38,6 +38,23 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { mustFinalize(t, &db2) } +func TestSaveStateRenamedRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WritingFile, WithRecovery(true), OpenDmsArgs{})) + require.NoError(t, db.SaveStateRenamed(t.Context(), "schemas.foo", "main.new", map[string]string{"name": "new"}, nil, map[string]any{"name": "old"})) + mustFinalize(t, &db) + + var db2 DeploymentState + require.NoError(t, db2.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) + entry, ok := db2.GetResourceEntry("schemas.foo") + require.True(t, ok) + assert.Equal(t, "main.new", entry.ID) + assert.Equal(t, map[string]any{"name": "old"}, entry.RenamedFrom) + mustFinalize(t, &db2) +} + func TestFinalizeWithoutResourceWrites(t *testing.T) { for _, mode := range []Mode{WritingFile, WritingDeploymentHistoryShadow} { path := filepath.Join(t.TempDir(), "state.json") From 8496ce6bced4f668f366c692defbfeb45f7f55da Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 6 Nov 2026 12:10:50 +0100 Subject: [PATCH 17/75] direct: find a moved schema or volume by its stable ID instead of renamed_from When the read at the recorded ID finds nothing, the plan reads at the ID the config gives the resource and takes what it finds only if its stable ID (schema_id, volume_id, now saved in the state entry and the DMS envelope) matches. This covers a deploy that stopped after renaming the parent and a rename done out of band, and never adopts a different object of the same name. It replaces renamed_from, which missed repeated renames and could not tell a rename from a recreate. The test server now deletes children with a forced catalog/schema delete and rejects a catalog create under a taken name, like UC. The migrate fixtures dropped the out-of-band catalog create that the scripts already run. Co-authored-by: Isaac --- acceptance/bin/print_state.py | 5 +- .../terraform-requests.json | 7 - .../terraform-requests.json | 7 - .../schemas/rename_interrupted/output.txt | 124 ++++++++---------- .../schemas/rename_interrupted/script | 8 +- bundle/deployplan/plan.go | 2 + bundle/direct/apply.go | 35 +++-- bundle/direct/bundle_plan.go | 94 +++++++++---- bundle/direct/bundle_plan_test.go | 4 +- bundle/direct/dresources/README.md | 4 + bundle/direct/dresources/adapter.go | 60 +++++++++ bundle/direct/dresources/schema.go | 10 ++ bundle/direct/dresources/volume.go | 10 ++ bundle/direct/dstate/dms.go | 14 +- bundle/direct/dstate/dms_test.go | 10 +- bundle/direct/dstate/state.go | 30 ++--- bundle/direct/dstate/state_test.go | 8 +- 17 files changed, 258 insertions(+), 174 deletions(-) diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 9badc1e4d5e..e7f69ccda56 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -146,6 +146,7 @@ def get_resources(target): "id": resource.get("resource_id"), "state": envelope.get("state") or {}, "depends_on": envelope.get("depends_on") or [], + "stable_id": envelope.get("stable_id"), } page_token = listed.get("next_page_token") if not page_token: @@ -154,12 +155,14 @@ def get_resources(target): def get_recorded_state(target): """The recorded resources in the on-disk state file's `state` shape (resources. -> - {__id__, state, depends_on}), so a recording run prints the same shape as a non-recording one.""" + {__id__, state, depends_on, stable_id}), so a recording run prints the same shape as a non-recording one.""" state = {} for key, value in sorted(get_resources(target).items()): entry = {"__id__": value["id"], "state": value["state"]} if value["depends_on"]: entry["depends_on"] = value["depends_on"] + if value["stable_id"]: + entry["stable_id"] = value["stable_id"] state[f"resources.{key}"] = entry return state diff --git a/acceptance/bundle/migrate/auto/registered-model-catalog-change/terraform-requests.json b/acceptance/bundle/migrate/auto/registered-model-catalog-change/terraform-requests.json index 6f4620d81f5..6c27b1fc729 100644 --- a/acceptance/bundle/migrate/auto/registered-model-catalog-change/terraform-requests.json +++ b/acceptance/bundle/migrate/auto/registered-model-catalog-change/terraform-requests.json @@ -1,11 +1,4 @@ [ - { - "method": "POST", - "path": "/api/2.1/unity-catalog/catalogs", - "body": { - "name": "othercat_[UNIQUE_NAME]" - } - }, { "method": "POST", "path": "/api/2.1/unity-catalog/models", diff --git a/acceptance/bundle/migrate/auto/schema-catalog-change/terraform-requests.json b/acceptance/bundle/migrate/auto/schema-catalog-change/terraform-requests.json index eda99b84d42..e7072606fd3 100644 --- a/acceptance/bundle/migrate/auto/schema-catalog-change/terraform-requests.json +++ b/acceptance/bundle/migrate/auto/schema-catalog-change/terraform-requests.json @@ -1,11 +1,4 @@ [ - { - "method": "POST", - "path": "/api/2.1/unity-catalog/catalogs", - "body": { - "name": "othercat_[UNIQUE_NAME]" - } - }, { "method": "POST", "path": "/api/2.1/unity-catalog/schemas", diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt index 091a96903cb..b2a40eebbfe 100644 --- a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt +++ b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt @@ -29,88 +29,68 @@ Exit code: [KILLED] } } -=== The next plan still re-keys the volume, whose state has the old schema +=== The volume is gone from its recorded path, so the plan looks for it at its path in config +>>> print_state.py +{ + "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", + "stable_id": "[VOLUME_ID]" +} + >>> [CLI] bundle plan --output json { - "resources.schemas.schema1": { - "action": "skip", - "remote_state": { - "browse_only": false, + "id": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "comment": "", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "custom_max_retention_hours": 0, - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "test-schema-renamed-[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][1], - "updated_by": "[USERNAME]" - }, - "changes": { - "comment": { - "action": "skip", - "reason": "empty", - "remote": "" - }, - "custom_max_retention_hours": { - "action": "skip", - "reason": "empty", - "remote": 0 - } + "comment": "c2", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" } }, - "resources.volumes.volume1": { - "depends_on": [ - { - "node": "resources.schemas.schema1", - "label": "${resources.schemas.schema1.name}" - } - ], - "action": "update_id", - "new_state": { - "value": { - "catalog_name": "main", - "comment": "c2", - "name": "myvolume", - "schema_name": "test-schema-renamed-[UNIQUE_NAME]", - "volume_type": "MANAGED" - } + "remote_state": { + "catalog_name": "main", + "comment": "c1", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[VOLUME_ID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "comment": { + "action": "update", + "old": "c1", + "new": "c2", + "remote": "c1" }, - "changes": { - "comment": { - "action": "update", - "old": "c1", - "new": "c2" - }, - "schema_name": { - "action": "update_id", - "reason": "parent_renamed", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]" - } + "schema_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-renamed-[UNIQUE_NAME]" + }, + "storage_location": { + "action": "skip", + "reason": "immutable_backend_value", + "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[VOLUME_ID]" } } } -=== It can, because the schema's old name was saved together with its rename ->>> print_state.py -{ - "__id__": "main.test-schema-renamed-[UNIQUE_NAME]", - "renamed_from": { - "name": "test-schema-[UNIQUE_NAME]" - } -} - === The next deploy updates the volume at its new path >>> [CLI] bundle deploy --force-lock Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/script b/acceptance/bundle/resources/schemas/rename_interrupted/script index ca0d499bb7e..196ee5d5faa 100644 --- a/acceptance/bundle/resources/schemas/rename_interrupted/script +++ b/acceptance/bundle/resources/schemas/rename_interrupted/script @@ -23,11 +23,9 @@ kill_after.py "PATCH /api/2.1/unity-catalog/volumes/$NEW_VOLUME" 0 1 trace errcode $CLI bundle deploy -q trace print_requests.py //unity-catalog -title "The next plan still re-keys the volume, whose state has the old schema" -trace $CLI bundle plan --output json | jq .plan - -title "It can, because the schema's old name was saved together with its rename" -trace print_state.py | jq '.state["resources.schemas.schema1"] | {__id__, renamed_from}' +title "The volume is gone from its recorded path, so the plan looks for it at its path in config" +trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, stable_id}' +trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"]' title "The next deploy updates the volume at its new path" trace $CLI bundle deploy --force-lock diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index f8055e51dd1..99d70f7b80b 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -119,6 +119,8 @@ func LoadPlanFromFile(path string) (*Plan, error) { } type PlanEntry struct { + // ID is set when the resource was found at another ID than the one in state: a rename + // moved it there (see findMoved). Applying the entry addresses the resource at ID. ID string `json:"id,omitempty"` DependsOn []DependsOnEntry `json:"depends_on,omitempty"` Action ActionType `json:"action,omitempty"` diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 2e7bb91b656..e1e718d02b4 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -44,6 +44,10 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, if oldID == "" { return errors.New("state entry not found") } + if planEntry.ID != "" { + // The plan found the resource at the ID its config gives it: a rename moved it. + oldID = planEntry.ID + } switch actionType { case deployplan.Recreate: @@ -89,7 +93,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn, nil) + err = d.saveState(ctx, db, newID, newState, d.DependsOn, d.Adapter.StableID(d.RemoteState)) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -187,7 +191,7 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("deleting state id=%s: %w", id, err) } } else { - err = d.saveState(ctx, db, id, newState, d.DependsOn, nil) + err = d.saveState(ctx, db, id, newState, d.DependsOn, d.stableID(planEntry.RemoteState)) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -236,7 +240,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn, renamedFrom(planEntry.Changes)) + err = d.saveState(ctx, db, newID, newState, d.DependsOn, d.stableID(planEntry.RemoteState)) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -329,7 +333,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = d.saveState(ctx, db, id, newState, d.DependsOn, nil) + err = d.saveState(ctx, db, id, newState, d.DependsOn, d.stableID(entry.RemoteState)) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -341,7 +345,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, // with content hashes, see dresources.CompactState) before persisting it. Fields already // smaller than a hash placeholder are persisted as is. Sensitive fields are replaced by a // placeholder value so secrets are never written to disk in plaintext. -func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry, renamedFrom map[string]any) error { +func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry, stableID string) error { compacted, err := dresources.CompactState(d.Adapter.ResourceConfig(), state) if err != nil { return fmt.Errorf("compacting state: %w", err) @@ -349,23 +353,16 @@ func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentSta if err := zeroSensitiveFields(d.Adapter, compacted); err != nil { return fmt.Errorf("redacting state: %w", err) } - return db.SaveStateRenamed(ctx, d.ResourceKey, newID, compacted, dependsOn, renamedFrom) + return db.SaveStateWithStableID(ctx, d.ResourceKey, newID, compacted, dependsOn, stableID) } -// renamedFrom returns the old values of the fields an update_id changes, for the state entry's -// RenamedFrom. -func renamedFrom(changes deployplan.Changes) map[string]any { - var result map[string]any - for path, change := range changes { - if change.Action != deployplan.UpdateWithID { - continue - } - if result == nil { - result = make(map[string]any) - } - result[path] = change.Old +// stableID returns the stable ID (see dstate.ResourceEntry) of the object being saved: from the +// remote state the operation returned, else from planRemote, the same object as the plan read it. +func (d *DeploymentUnit) stableID(planRemote any) string { + if id := d.Adapter.StableID(d.RemoteState); id != "" { + return id } - return result + return d.Adapter.StableID(planRemote) } func parseState(destType reflect.Type, raw json.RawMessage) (any, error) { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 1eb89c6499f..5eeaaf86570 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -334,6 +334,13 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } } + if remoteState == nil { + entry.ID, remoteState, err = findMoved(ctx, adapter, sv, dbentry) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } + } // We have a choice whether to include remoteState or remoteStateComparable from below. // Including remoteState because in the near future remoteState is expected to become a superset struct of remoteStateComparable @@ -377,20 +384,23 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } - followedRename, err := followParentRenames(adapter, entry.Changes, parentRenames) + err = followParentRenames(adapter, entry.Changes, parentRenames) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: classifying changes: %w", errorPrefix, err)) return false } - if remoteState == nil && !followedRename { + if remoteState == nil { // Even if local action is "recreate" which is higher than "create", we should still pick "create" here - // because we know remote does not exist. A child that followed its parent's rename is not - // missing: it moved, and the read used its old ID. + // because we know remote does not exist. action = deployplan.Create } else { action = getMaxAction(entry.Changes) } + if entry.ID != "" { + // Found at a new ID: the state has to follow it there. + action = deployplan.GetHigherAction(action, deployplan.UpdateWithID) + } // Note, this unconditionally stores remoteState. However, it may updated post-deploy, so whether // it can be used for variable resolution depends on several factors, see canReadRemoteCache in LookupReferencePreDeploy @@ -712,9 +722,6 @@ func (b *DeploymentBundle) parentRenames(resourceKey string) (map[string]*deploy } // renameChange returns the change to field of resourceKey if it is part of an update_id, nil otherwise. -// -// When this plan does not rename the field, the rename recorded with the resource's state counts -// too: a deploy interrupted after renaming the parent leaves its children under the old name. func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan.ChangeDesc, error) { entry, err := b.Plan.ReadLockEntry(resourceKey) if err != nil { @@ -723,25 +730,60 @@ func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan. defer b.Plan.ReadUnlockEntry(resourceKey) // A missing entry is reported by resolveReferences. - if entry == nil { + if entry == nil || entry.Action != deployplan.UpdateWithID { return nil, nil } - if change := entry.Changes[field]; change != nil && change.Action == deployplan.UpdateWithID { - return change, nil - } - // A parent that is created or recreated is a new object; UC does not move children into it. - if !entry.Action.KeepsID() && entry.Action != deployplan.UpdateWithID { + change := entry.Changes[field] + if change == nil || change.Action != deployplan.UpdateWithID { return nil, nil } - dbentry, ok := b.StateDB.GetResourceEntry(resourceKey) - if !ok { - return nil, nil + return change, nil +} + +// findMoved looks for a resource missing from its recorded ID at the ID its config gives it, where a +// rename can have moved it: the resource or its parent renamed out of band, or by a deploy that +// stopped before re-keying it. What is found there is taken only if its stable ID matches the +// recorded one, so an unrelated object with the same name is never adopted; the create then fails +// on it instead. +func findMoved(ctx context.Context, adapter *dresources.Adapter, sv *structvar.StructVar, dbentry dstate.ResourceEntry) (string, any, error) { + if !adapter.HasStableID() || !adapter.HasDoUpdateWithID() || dbentry.StableID == "" || hasUnresolvedIDField(adapter, sv) { + return "", nil, nil + } + id, err := adapter.IDFromState(sv.Value) + if err != nil || id == dbentry.ID { + return "", nil, err + } + remote, err := retry.With(ctx, retry.IsTransientRead, func() (any, error) { + return adapter.DoRead(ctx, id) + }) + if apierr.IsMissing(err) { + return "", nil, nil } - old, ok := dbentry.RenamedFrom[field] - if !ok { - return nil, nil + if err != nil { + return "", nil, fmt.Errorf("reading id=%q: %w", id, err) } - return &deployplan.ChangeDesc{Action: deployplan.UpdateWithID, Old: old}, nil + if adapter.StableID(remote) != dbentry.StableID { + return "", nil, nil + } + return id, remote, nil +} + +// hasUnresolvedIDField reports whether a field the ID is built from is still an unresolved reference. +func hasUnresolvedIDField(adapter *dresources.Adapter, sv *structvar.StructVar) bool { + cfg := adapter.ResourceConfig() + for pathString := range sv.Refs { + path, err := structpath.ParsePath(pathString) + if err != nil { + return true + } + if _, ok := findMatchingRule(path, cfg.ProvidedIDFields); ok { + return true + } + if _, ok := findMatchingRule(path, cfg.UpdatableIDFields); ok { + return true + } + } + return false } // followParentRenames plans an update_id instead of a recreate for an ID field that changes @@ -751,13 +793,10 @@ func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan. // // The old values must match too: a child moved to the renamed parent from somewhere else // still has to be recreated. -// -// It reports whether any change follows a rename. -func followParentRenames(adapter *dresources.Adapter, changes deployplan.Changes, parentRenames map[string]*deployplan.ChangeDesc) (bool, error) { +func followParentRenames(adapter *dresources.Adapter, changes deployplan.Changes, parentRenames map[string]*deployplan.ChangeDesc) error { if !adapter.HasDoUpdateWithID() { - return false, nil + return nil } - followed := false for pathString, parentChange := range parentRenames { ch := changes[pathString] if ch == nil || ch.Action != deployplan.Recreate || !structdiff.IsEqual(ch.Old, parentChange.Old) { @@ -765,16 +804,15 @@ func followParentRenames(adapter *dresources.Adapter, changes deployplan.Changes } path, err := structpath.ParsePath(pathString) if err != nil { - return false, err + return err } if _, ok := findMatchingRule(path, adapter.ResourceConfig().ProvidedIDFields); !ok { continue } ch.Action = deployplan.UpdateWithID ch.Reason = deployplan.ReasonParentRenamed - followed = true } - return followed, nil + return nil } // shouldSkipNormalized skips a change that is a false diff caused by UC API diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 7d735755080..f50de986918 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -323,10 +323,8 @@ func TestFollowParentRenames(t *testing.T) { parentRenames := map[string]*deployplan.ChangeDesc{ tt.field: {Action: deployplan.UpdateWithID, Reason: "id_changes", Old: "s_old", New: "s_new", Remote: "s_old"}, } - followed, err := followParentRenames(adapter, deployplan.Changes{tt.field: ch}, parentRenames) - require.NoError(t, err) + require.NoError(t, followParentRenames(adapter, deployplan.Changes{tt.field: ch}, parentRenames)) assert.Equal(t, tt.expectedAction, ch.Action) - assert.Equal(t, tt.expectedAction == deployplan.UpdateWithID, followed) }) } } diff --git a/bundle/direct/dresources/README.md b/bundle/direct/dresources/README.md index 78952537943..98f10f85228 100644 --- a/bundle/direct/dresources/README.md +++ b/bundle/direct/dresources/README.md @@ -47,6 +47,10 @@ For resources whose create or update is asynchronous (the resource is not immedi If the API may return a slice's elements in a different order between calls (e.g., `depends_on` in job tasks, `privileges` in grants), implement `KeyedSlices` to compare elements by a natural key rather than by index. Without this, every deploy after any reordering shows phantom diffs. +## Moved resources: IDFromState / StableID + +If the ID is built from the resource's own fields (a UC full name) and the backend has an ID that a rename keeps (a volume's `volume_id`), implement `IDFromState` and `StableID` together, along with `DoUpdateWithID`. When nothing is at the recorded ID, the plan reads at the ID the config gives the resource and takes what it finds only if its stable ID matches the one saved in state. This covers a rename done out of band, or by a deploy that stopped before re-keying the resource. + ## Empty states: IsEmptyState If a desired state describes no resource at all (e.g. an empty grants list), implement `IsEmptyState`. The planner omits such a node instead of planning a create, and apply drops its state entry instead of persisting one. diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 80ece24b2be..75c91cdc4d5 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -118,6 +118,17 @@ type IResource interface { // (returned by GET, not 404) that rejects a second DELETE. // Example: func (*ResourceApp) IsGone(remote *AppRemote) bool IsGone(remoteState any) bool + + // [Optional] IDFromState returns the ID the state gives the resource, for resource types whose ID + // is built from their own fields (a UC full name). Implemented together with StableID. + // Example: func (*ResourceVolume) IDFromState(state *catalog.CreateVolumeRequestContent) string + IDFromState(state any) string + + // [Optional] StableID returns the backend's ID for the object that a rename keeps (a volume's + // volume_id). When the resource is not at its recorded ID, the plan looks for it at the ID + // IDFromState gives it, and takes what it finds only if StableID matches the recorded one. + // Example: func (*ResourceVolume) StableID(remote *catalog.VolumeInfo) string + StableID(remoteState any) string } // Adapter wraps resource implementation, validates signatures and type consistency across methods @@ -141,6 +152,8 @@ type Adapter struct { overrideChangeDesc *calladapt.BoundCaller doResize *calladapt.BoundCaller isGone *calladapt.BoundCaller + idFromState *calladapt.BoundCaller + stableID *calladapt.BoundCaller resourceConfig *ResourceLifecycleConfig generatedResourceConfig *ResourceLifecycleConfig @@ -182,6 +195,8 @@ func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceC waitAfterDelete: nil, overrideChangeDesc: nil, isGone: nil, + idFromState: nil, + stableID: nil, resourceConfig: GetResourceConfig(resourceType), generatedResourceConfig: GetGeneratedResourceConfig(resourceType), keyedSlices: nil, @@ -308,6 +323,16 @@ func (a *Adapter) initMethods(resource any) error { return err } + a.idFromState, err = calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "IDFromState") + if err != nil { + return err + } + + a.stableID, err = calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "StableID") + if err != nil { + return err + } + keyedSlicesCall, err := calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "KeyedSlices") if err != nil { return err @@ -402,6 +427,14 @@ func (a *Adapter) validate() error { validations = append(validations, "IsGone remoteState", a.isGone.InTypes[0], remoteType) } + if (a.idFromState == nil) != (a.stableID == nil) { + return errors.New("IDFromState and StableID must be implemented together") + } + if a.idFromState != nil { + validations = append(validations, "IDFromState state", a.idFromState.InTypes[0], stateType) + validations = append(validations, "StableID remoteState", a.stableID.InTypes[0], remoteType) + } + if a.doUpdateWithID != nil { validations = append(validations, "DoUpdateWithID newState", a.doUpdateWithID.InTypes[2], stateType) // DoUpdateWithID must return (string, remoteType, error) @@ -727,6 +760,33 @@ func (a *Adapter) IsGone(remoteState any) bool { return outs[0].(bool) } +// HasStableID reports whether the resource implements IDFromState and StableID. +func (a *Adapter) HasStableID() bool { + return a.stableID != nil +} + +// IDFromState returns the ID the state gives the resource; see IResource.IDFromState. +func (a *Adapter) IDFromState(state any) (string, error) { + outs, err := a.idFromState.Call(state) + if err != nil { + return "", err + } + return outs[0].(string), nil +} + +// StableID returns the rename-proof ID of the object in remoteState; see IResource.StableID. +// It is empty for a resource type without one and for a nil remote state. +func (a *Adapter) StableID(remoteState any) string { + if a.stableID == nil || remoteState == nil || reflect.ValueOf(remoteState).IsNil() { + return "" + } + outs, err := a.stableID.Call(remoteState) + if err != nil { + return "" + } + return outs[0].(string) +} + // prepareCallRequired prepares a call and ensures the method is found. func prepareCallRequired(resource any, methodName string) (*calladapt.BoundCaller, error) { caller, err := calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), methodName) diff --git a/bundle/direct/dresources/schema.go b/bundle/direct/dresources/schema.go index 8a928cb8989..6e1856aa421 100644 --- a/bundle/direct/dresources/schema.go +++ b/bundle/direct/dresources/schema.go @@ -120,6 +120,16 @@ func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config * return response.FullName, response, nil } +// IDFromState returns the full name, which is the ID. +func (*ResourceSchema) IDFromState(state *catalog.CreateSchema) string { + return state.CatalogName + "." + state.Name +} + +// StableID returns the schema_id, which a rename keeps. +func (*ResourceSchema) StableID(remote *catalog.SchemaInfo) string { + return remote.SchemaId +} + func (r *ResourceSchema) DoDelete(ctx context.Context, id string, _ *catalog.CreateSchema) error { return r.client.Schemas.Delete(ctx, catalog.DeleteSchemaRequest{ FullName: id, diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 445cd826314..06674d45f96 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -121,6 +121,16 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * return response.FullName, response, nil } +// IDFromState returns the full name, which is the ID. +func (*ResourceVolume) IDFromState(state *catalog.CreateVolumeRequestContent) string { + return state.CatalogName + "." + state.SchemaName + "." + state.Name +} + +// StableID returns the volume_id, which a rename keeps. +func (*ResourceVolume) StableID(remote *catalog.VolumeInfo) string { + return remote.VolumeId +} + func (r *ResourceVolume) DoDelete(ctx context.Context, id string, _ *catalog.CreateVolumeRequestContent) error { return r.client.Volumes.DeleteByName(ctx, id) } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 3f38ddc2bd8..4842b8de10b 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -12,9 +12,9 @@ import ( // config so depends_on survives the round trip: DMS has no field for dependency edges, and // nesting them in the config would collide with resource fields of the same name. type RecordedState struct { - State json.RawMessage `json:"state"` - DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` - RenamedFrom map[string]any `json:"renamed_from,omitempty"` + State json.RawMessage `json:"state"` + DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` + StableID string `json:"stable_id,omitempty"` } // applyDMSState fills in the resource state from what DMS recorded. For a deployment that uses DMS @@ -55,9 +55,9 @@ func stateEntry(res dms.Resource) (ResourceEntry, error) { } return ResourceEntry{ - ID: res.ID, - State: recorded.State, - DependsOn: recorded.DependsOn, - RenamedFrom: recorded.RenamedFrom, + ID: res.ID, + State: recorded.State, + DependsOn: recorded.DependsOn, + StableID: recorded.StableID, }, nil } diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index 18a1aceddcc..0dc86caf879 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -45,15 +45,15 @@ func TestApplyDMSState(t *testing.T) { }, }, { - name: "renamed_from comes back from the envelope", + name: "stable_id comes back from the envelope", recorded: []dms.Resource{ - {Key: "resources.schemas.foo", ID: "main.new", State: `{"state":{"name":"new"},"renamed_from":{"name":"old"}}`}, + {Key: "resources.schemas.foo", ID: "main.foo", State: `{"state":{"name":"foo"},"stable_id":"uuid-1"}`}, }, want: map[string]ResourceEntry{ "resources.schemas.foo": { - ID: "main.new", - State: json.RawMessage(`{"name":"new"}`), - RenamedFrom: map[string]any{"name": "old"}, + ID: "main.foo", + State: json.RawMessage(`{"name":"foo"}`), + StableID: "uuid-1", }, }, }, diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index ace32a78770..bd8bfc97e23 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -170,11 +170,10 @@ type ResourceEntry struct { State json.RawMessage `json:"state"` DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` - // RenamedFrom holds the old values of the fields an in-place ID change (update_id) changed, - // keyed by field path, until the resource is saved again. It is saved in the same write as - // the new ID, so a child that an interrupted deploy left under the old name can still follow - // the rename. - RenamedFrom map[string]any `json:"renamed_from,omitempty"` + // StableID is the backend's ID for the object that survives a rename (a volume's volume_id), + // for resource types that have one. When the object is not at ID any more, the plan looks for + // it at the ID its config gives it and takes what it finds there only if StableID matches. + StableID string `json:"stable_id,omitempty"` } type WALEntry struct { @@ -295,7 +294,7 @@ func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause e var recorded json.RawMessage if entry, ok := db.GetResourceEntry(resourceKey); resourceID != "" && ok && len(entry.State) > 0 { var err error - recorded, err = json.Marshal(RecordedState{State: entry.State, DependsOn: entry.DependsOn, RenamedFrom: entry.RenamedFrom}) + recorded, err = json.Marshal(RecordedState{State: entry.State, DependsOn: entry.DependsOn, StableID: entry.StableID}) if err != nil { // Nothing the caller can act on, so record the failure without the state. recorded = nil @@ -321,12 +320,11 @@ func NewDatabase(lineage string, serial int) Database { // SaveState records the resource's state after an operation was applied to it. func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { - return db.SaveStateRenamed(ctx, key, newID, state, dependsOn, nil) + return db.SaveStateWithStableID(ctx, key, newID, state, dependsOn, "") } -// SaveStateRenamed is SaveState for an ID changed in place: renamedFrom (see ResourceEntry) is -// recorded together with the new ID. -func (db *DeploymentState) SaveStateRenamed(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry, renamedFrom map[string]any) error { +// SaveStateWithStableID is SaveState that also records the object's stable ID (see ResourceEntry). +func (db *DeploymentState) SaveStateWithStableID(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry, stableID string) error { db.AssertOpenedForWrite() db.mu.Lock() defer db.mu.Unlock() @@ -340,10 +338,10 @@ func (db *DeploymentState) SaveStateRenamed(ctx context.Context, key, newID stri return err } entry := ResourceEntry{ - ID: newID, - State: json.RawMessage(jsonMessage), - DependsOn: dependsOn, - RenamedFrom: renamedFrom, + ID: newID, + State: json.RawMessage(jsonMessage), + DependsOn: dependsOn, + StableID: stableID, } switch db.mode { @@ -357,7 +355,7 @@ func (db *DeploymentState) SaveStateRenamed(ctx context.Context, key, newID stri db.Data.State[key] = entry db.stateIDs[key] = newID if buf := db.operationBuffer; buf != nil { - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, RenamedFrom: renamedFrom}) + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, StableID: stableID}) if err != nil { return err } @@ -372,7 +370,7 @@ func (db *DeploymentState) SaveStateRenamed(ctx context.Context, key, newID stri } db.stateIDs[key] = newID if buf := db.operationBuffer; buf != nil { - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, RenamedFrom: renamedFrom}) + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, StableID: stableID}) if err != nil { return err } diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 9e22d15b8a1..6330a5d99e4 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -38,20 +38,20 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { mustFinalize(t, &db2) } -func TestSaveStateRenamedRoundTrip(t *testing.T) { +func TestSaveStateWithStableIDRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WritingFile, WithRecovery(true), OpenDmsArgs{})) - require.NoError(t, db.SaveStateRenamed(t.Context(), "schemas.foo", "main.new", map[string]string{"name": "new"}, nil, map[string]any{"name": "old"})) + require.NoError(t, db.SaveStateWithStableID(t.Context(), "schemas.foo", "main.foo", map[string]string{"name": "foo"}, nil, "uuid-1")) mustFinalize(t, &db) var db2 DeploymentState require.NoError(t, db2.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) entry, ok := db2.GetResourceEntry("schemas.foo") require.True(t, ok) - assert.Equal(t, "main.new", entry.ID) - assert.Equal(t, map[string]any{"name": "old"}, entry.RenamedFrom) + assert.Equal(t, "main.foo", entry.ID) + assert.Equal(t, "uuid-1", entry.StableID) mustFinalize(t, &db2) } From e913e1799fe5894b41011193da17c117ad2a3b5a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 6 Nov 2026 12:11:04 +0100 Subject: [PATCH 18/75] acc: out-of-band renames and recreates of volumes, schemas and catalogs Co-authored-by: Isaac --- .../databricks.yml.tmpl | 16 ++ .../rename_all_out_of_band/out.test.toml | 2 + .../rename_all_out_of_band/output.txt | 174 ++++++++++++++++++ .../catalogs/rename_all_out_of_band/script | 44 +++++ .../catalogs/rename_all_out_of_band/test.toml | 7 + .../rename_out_of_band/databricks.yml.tmpl | 16 ++ .../catalogs/rename_out_of_band/out.test.toml | 2 + .../catalogs/rename_out_of_band/output.txt | 139 ++++++++++++++ .../catalogs/rename_out_of_band/script | 40 ++++ .../catalogs/rename_out_of_band/test.toml | 7 + .../recreate_interrupted/databricks.yml.tmpl | 14 ++ .../recreate_interrupted/out.test.toml | 2 + .../schemas/recreate_interrupted/output.txt | 121 ++++++++++++ .../schemas/recreate_interrupted/script | 36 ++++ .../schemas/recreate_interrupted/test.toml | 30 +++ .../recreate_out_of_band/databricks.yml.tmpl | 13 ++ .../recreate_out_of_band/out.test.toml | 2 + .../schemas/recreate_out_of_band/output.txt | 91 +++++++++ .../schemas/recreate_out_of_band/script | 38 ++++ .../schemas/recreate_out_of_band/test.toml | 7 + .../rename_out_of_band/databricks.yml.tmpl | 13 ++ .../schemas/rename_out_of_band/out.test.toml | 2 + .../schemas/rename_out_of_band/output.txt | 152 +++++++++++++++ .../schemas/rename_out_of_band/script | 44 +++++ .../schemas/rename_out_of_band/test.toml | 7 + .../recreate_out_of_band/databricks.yml.tmpl | 13 ++ .../recreate_out_of_band/out.test.toml | 2 + .../volumes/recreate_out_of_band/output.txt | 81 ++++++++ .../volumes/recreate_out_of_band/script | 34 ++++ .../volumes/recreate_out_of_band/test.toml | 7 + .../rename_out_of_band/databricks.yml.tmpl | 13 ++ .../volumes/rename_out_of_band/out.test.toml | 2 + .../volumes/rename_out_of_band/output.txt | 121 ++++++++++++ .../volumes/rename_out_of_band/script | 41 +++++ .../volumes/rename_out_of_band/test.toml | 7 + 35 files changed, 1340 insertions(+) create mode 100644 acceptance/bundle/resources/catalogs/rename_all_out_of_band/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml create mode 100644 acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt create mode 100644 acceptance/bundle/resources/catalogs/rename_all_out_of_band/script create mode 100644 acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml create mode 100644 acceptance/bundle/resources/catalogs/rename_out_of_band/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml create mode 100644 acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt create mode 100644 acceptance/bundle/resources/catalogs/rename_out_of_band/script create mode 100644 acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml create mode 100644 acceptance/bundle/resources/schemas/recreate_interrupted/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/recreate_interrupted/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/recreate_interrupted/output.txt create mode 100644 acceptance/bundle/resources/schemas/recreate_interrupted/script create mode 100644 acceptance/bundle/resources/schemas/recreate_interrupted/test.toml create mode 100644 acceptance/bundle/resources/schemas/recreate_out_of_band/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/recreate_out_of_band/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt create mode 100644 acceptance/bundle/resources/schemas/recreate_out_of_band/script create mode 100644 acceptance/bundle/resources/schemas/recreate_out_of_band/test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_out_of_band/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/rename_out_of_band/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_out_of_band/output.txt create mode 100644 acceptance/bundle/resources/schemas/rename_out_of_band/script create mode 100644 acceptance/bundle/resources/schemas/rename_out_of_band/test.toml create mode 100644 acceptance/bundle/resources/volumes/recreate_out_of_band/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/volumes/recreate_out_of_band/out.test.toml create mode 100644 acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt create mode 100644 acceptance/bundle/resources/volumes/recreate_out_of_band/script create mode 100644 acceptance/bundle/resources/volumes/recreate_out_of_band/test.toml create mode 100644 acceptance/bundle/resources/volumes/rename_out_of_band/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/volumes/rename_out_of_band/out.test.toml create mode 100644 acceptance/bundle/resources/volumes/rename_out_of_band/output.txt create mode 100644 acceptance/bundle/resources/volumes/rename_out_of_band/script create mode 100644 acceptance/bundle/resources/volumes/rename_out_of_band/test.toml diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..e45eccddc44 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + volumes: + volume1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt new file mode 100644 index 00000000000..7b1f43ba6ce --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt @@ -0,0 +1,174 @@ + +=== Deploy a catalog with a schema and a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the volume, the schema and the catalog out of band +>>> [CLI] volumes update test_catalog_[UNIQUE_NAME].myschema.myvolume --new-name myvolume2 +{ + "full_name": "test_catalog_[UNIQUE_NAME].myschema.myvolume2" +} + +>>> [CLI] schemas update test_catalog_[UNIQUE_NAME].myschema --new-name myschema2 +{ + "full_name": "test_catalog_[UNIQUE_NAME].myschema2" +} + +>>> [CLI] catalogs update test_catalog_[UNIQUE_NAME] --new-name test_catalog_renamed_[UNIQUE_NAME] +{ + "name": "test_catalog_renamed_[UNIQUE_NAME]" +} + +=== Rename all three in the config to match +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +>>> update_file.py databricks.yml name: myschema name: myschema2 + +>>> update_file.py databricks.yml name: myvolume name: myvolume2 + +=== The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "create", + "new_state": { + "value": { + "name": "test_catalog_renamed_[UNIQUE_NAME]" + } + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]" + } + } + }, + "resources.schemas.schema1": { + "id": "test_catalog_renamed_[UNIQUE_NAME].myschema2", + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2", + "name": "myschema2", + "owner": "[USERNAME]", + "schema_id": "[SCHEMA_ID]" + }, + "changes": { + "catalog_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "name": { + "action": "skip", + "reason": "remote_already_set", + "old": "myschema", + "new": "myschema2", + "remote": "myschema2" + } + } + }, + "resources.volumes.volume1": { + "id": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume2", + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myvolume2", + "schema_name": "myschema2", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume2", + "name": "myvolume2", + "owner": "[USERNAME]", + "schema_name": "myschema2", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "catalog_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "name": { + "action": "skip", + "reason": "remote_already_set", + "old": "myvolume", + "new": "myvolume2", + "remote": "myvolume2" + }, + "schema_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "myschema", + "new": "myschema2", + "remote": "myschema2" + } + } + } +} + +=== The catalog's create fails because the renamed catalog has its name +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Error: cannot create resources.catalogs.catalog1: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists (400 CATALOG_ALREADY_EXISTS) + +Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/catalogs +HTTP Status: 400 Bad Request +API error_code: CATALOG_ALREADY_EXISTS +API message: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists + +Error: cannot update_id resources.schemas.schema1: dependency failed: resources.catalogs.catalog1 + +Error: cannot update_id resources.volumes.volume1: dependency failed: resources.catalogs.catalog1 + +Files: 3 uploaded, 0 deleted + +>>> errcode [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 0 deleted + +>>> errcode [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script new file mode 100644 index 00000000000..9645fc0dc02 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script @@ -0,0 +1,44 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_CATALOG="test_catalog_${UNIQUE_NAME}" +NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" + +cleanup() { + trace errcode $CLI bundle destroy --auto-approve + # The renamed catalog is not where the state looks for it, so destroy cannot remove it. + trace errcode $CLI catalogs delete "$NEW_CATALOG" --force + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location +} + +title "Deploy a catalog with a schema and a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the volume, the schema and the catalog out of band" +trace $CLI volumes update "$OLD_CATALOG.myschema.myvolume" --new-name myvolume2 | jq "{full_name}" +trace $CLI schemas update "$OLD_CATALOG.myschema" --new-name myschema2 | jq "{full_name}" +trace $CLI catalogs update "$OLD_CATALOG" --new-name "$NEW_CATALOG" | jq "{name}" +print_requests.py //unity-catalog &> LOG.rename_requests + +title "Rename all three in the config to match" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ +trace update_file.py databricks.yml "name: myschema" "name: myschema2" +trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" +NEW_SCHEMA="$NEW_CATALOG.myschema2" +NEW_VOLUME="$NEW_SCHEMA.myvolume2" + +title "The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again" +plan + +title "The catalog's create fails because the renamed catalog has its name" +trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/rename_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..e45eccddc44 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + volumes: + volume1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml b/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt new file mode 100644 index 00000000000..09bf7199065 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt @@ -0,0 +1,139 @@ + +=== Deploy a catalog with a schema and a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the catalog out of band. UC moves the schema and the volume with it +>>> [CLI] catalogs update test_catalog_[UNIQUE_NAME] --new-name test_catalog_renamed_[UNIQUE_NAME] +{ + "name": "test_catalog_renamed_[UNIQUE_NAME]" +} + +=== Rename the catalog in the config to match +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +=== The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "create", + "new_state": { + "value": { + "name": "test_catalog_renamed_[UNIQUE_NAME]" + } + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]" + } + } + }, + "resources.schemas.schema1": { + "id": "test_catalog_renamed_[UNIQUE_NAME].myschema", + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myschema" + } + }, + "remote_state": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema", + "name": "myschema", + "owner": "[USERNAME]", + "schema_id": "[SCHEMA_ID]" + }, + "changes": { + "catalog_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" + } + } + }, + "resources.volumes.volume1": { + "id": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myvolume", + "schema_name": "myschema", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "myschema", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "catalog_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" + } + } + } +} + +=== The catalog's create fails because the renamed catalog has its name +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Error: cannot create resources.catalogs.catalog1: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists (400 CATALOG_ALREADY_EXISTS) + +Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/catalogs +HTTP Status: 400 Bad Request +API error_code: CATALOG_ALREADY_EXISTS +API message: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists + +Error: cannot update_id resources.schemas.schema1: dependency failed: resources.catalogs.catalog1 + +Error: cannot update_id resources.volumes.volume1: dependency failed: resources.catalogs.catalog1 + +Files: 3 uploaded, 0 deleted + +>>> errcode [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 0 deleted + +>>> errcode [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_out_of_band/script new file mode 100644 index 00000000000..8b161d76a37 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/script @@ -0,0 +1,40 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_CATALOG="test_catalog_${UNIQUE_NAME}" +NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" + +cleanup() { + trace errcode $CLI bundle destroy --auto-approve + # The renamed catalog is not where the state looks for it, so destroy cannot remove it. + trace errcode $CLI catalogs delete "$NEW_CATALOG" --force + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location +} + +title "Deploy a catalog with a schema and a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the catalog out of band. UC moves the schema and the volume with it" +trace $CLI catalogs update "$OLD_CATALOG" --new-name "$NEW_CATALOG" | jq "{name}" +print_requests.py //unity-catalog &> LOG.rename_requests + +title "Rename the catalog in the config to match" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ +NEW_SCHEMA="$NEW_CATALOG.myschema" +NEW_VOLUME="$NEW_SCHEMA.myvolume" + +title "The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again" +plan + +title "The catalog's create fails because the renamed catalog has its name" +trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/recreate_interrupted/databricks.yml.tmpl new file mode 100644 index 00000000000..a965908256c --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + storage_root: s3://mybucket/a + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/out.test.toml b/acceptance/bundle/resources/schemas/recreate_interrupted/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt b/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt new file mode 100644 index 00000000000..69ff54088bd --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt @@ -0,0 +1,121 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema and change its storage_root, which recreates it. The CLI is killed on the volume's recreate, after the schema's +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> update_file.py databricks.yml s3://mybucket/a s3://mybucket/b + +>>> errcode [CLI] bundle deploy -q --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.schema1 + +This action will result in the deletion or recreation of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + recreate resources.volumes.volume1 +[PROCESS_KILLED] + +Exit code: [KILLED] + +>>> print_requests.py //unity-catalog +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "q": { + "force": "true" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "test-schema-renamed-[UNIQUE_NAME]", + "storage_root": "s3://mybucket/b" + } +} + +=== The volume went with the old schema. Nothing with its volume_id is at its new path, so the plan creates it +>>> [CLI] bundle plan --output json +{ + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "changes": { + "schema_name": { + "action": "recreate", + "reason": "id_field", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]" + } + } +} + +=== The next deploy creates a new volume +>>> [CLI] bundle deploy --force-lock +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created volumes.volume1 +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +>>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "volume_id": "[UUID]" +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve --force-lock +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/script b/acceptance/bundle/resources/schemas/recreate_interrupted/script new file mode 100644 index 00000000000..df501ef6124 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/script @@ -0,0 +1,36 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_VOLUME="main.test-schema-${UNIQUE_NAME}.myvolume" +NEW_VOLUME="main.test-schema-renamed-${UNIQUE_NAME}.myvolume" + +cleanup() { + trace $CLI bundle destroy --auto-approve --force-lock + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI volumes read "$OLD_VOLUME" | jq -r .volume_id)" VOLUME_ID + +title "Rename the schema and change its storage_root, which recreates it. The CLI is killed on the volume's recreate, after the schema's" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +trace update_file.py databricks.yml s3://mybucket/a s3://mybucket/b +kill_after.py "DELETE /api/2.1/unity-catalog/volumes/$OLD_VOLUME" 0 1 +# -q: the per-resource lines race the kill signal. +trace errcode $CLI bundle deploy -q --auto-approve +trace print_requests.py //unity-catalog + +title "The volume went with the old schema. Nothing with its volume_id is at its new path, so the plan creates it" +trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"]' + +title "The next deploy creates a new volume" +trace $CLI bundle deploy --force-lock +trace print_requests.py //unity-catalog +trace $CLI volumes read "$NEW_VOLUME" | jq '{full_name, volume_id}' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/test.toml b/acceptance/bundle/resources/schemas/recreate_interrupted/test.toml new file mode 100644 index 00000000000..23d6ce8f7fa --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/test.toml @@ -0,0 +1,30 @@ +# Local-only: the test server kills the CLI mid-deploy (kill_after.py), which a real workspace +# cannot, and an explicit storage_root needs an external location on a real workspace. +Cloud = false + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] + +# KillCaller needs the CLI's pid. A kill leaves no state in DMS to recover from, so DMS is not run. +Env.DATABRICKS_CLI_TEST_PID = "1" +EnvMatrix.DMS = [""] + +# Normalize how a killed process shows up on each OS, as in bundle/deploy/wal. +[[Repls]] +Old = 'script: line \d+:\s+\d+ Killed(: 9)?\s+"\$@"' +New = '[PROCESS_KILLED]' + +[[Repls]] +Old = '(\n>>> errcode [^\n]+\n)\nExit code:' +New = "${1}[PROCESS_KILLED]\n\nExit code:" + +[[Repls]] +Old = 'Exit code: 137' +New = 'Exit code: [KILLED]' + +[[Repls]] +Old = '(\[PROCESS_KILLED\]\n\nExit code: )1' +New = '${1}[KILLED]' diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/recreate_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..2234fdf2e39 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/out.test.toml b/acceptance/bundle/resources/schemas/recreate_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt b/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt new file mode 100644 index 00000000000..f2520fb4368 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt @@ -0,0 +1,91 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Out of band, drop the schema and create another one under a new name, with a volume of the same name +>>> [CLI] schemas delete main.test-schema-[UNIQUE_NAME] --force + +=== With the config renamed to match, the plan finds a schema and a volume at the new names, but their ids differ, so it plans creates +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> print_state.py +{ + "resources.schemas.schema1": { + "__id__": "main.test-schema-[UNIQUE_NAME]", + "stable_id": "[SCHEMA_ID]" + }, + "resources.volumes.volume1": { + "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", + "stable_id": "[VOLUME_ID]" + } +} + +>>> [CLI] schemas get main.test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "schema_id": "[NEW_SCHEMA_ID]" +} + +>>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "volume_id": "[NEW_VOLUME_ID]" +} + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "test-schema-renamed-[UNIQUE_NAME]" + } + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "changes": { + "schema_name": { + "action": "recreate", + "reason": "id_field", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]" + } + } + } +} + +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 0 deleted + +>>> [CLI] schemas delete main.test-schema-renamed-[UNIQUE_NAME] --force diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/script b/acceptance/bundle/resources/schemas/recreate_out_of_band/script new file mode 100644 index 00000000000..2d0997ea594 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/script @@ -0,0 +1,38 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_SCHEMA="main.test-schema-${UNIQUE_NAME}" +NEW_SCHEMA="main.test-schema-renamed-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + # The schema created out of band is not in the state, so destroy does not remove it. + trace $CLI schemas delete "$NEW_SCHEMA" --force + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +} + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "$OLD_SCHEMA" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "$OLD_SCHEMA.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Out of band, drop the schema and create another one under a new name, with a volume of the same name" +trace $CLI schemas delete "$OLD_SCHEMA" --force +add_repl "$($CLI schemas create "test-schema-renamed-${UNIQUE_NAME}" main | jq -r .schema_id)" NEW_SCHEMA_ID +add_repl "$($CLI volumes create main "test-schema-renamed-${UNIQUE_NAME}" myvolume MANAGED | jq -r .volume_id)" NEW_VOLUME_ID +print_requests.py //unity-catalog &> LOG.recreate_requests + +title "With the config renamed to match, the plan finds a schema and a volume at the new names, but their ids differ, so it plans creates" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +trace print_state.py | jq '.state | map_values({__id__, stable_id})' +trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, schema_id}' +trace $CLI volumes read "$NEW_SCHEMA.myvolume" | jq '{full_name, volume_id}' +plan diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/test.toml b/acceptance/bundle/resources/schemas/recreate_out_of_band/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..2234fdf2e39 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/out.test.toml b/acceptance/bundle/resources/schemas/rename_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt b/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt new file mode 100644 index 00000000000..0b951a5e912 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt @@ -0,0 +1,152 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema out of band. UC moves the volume with it +>>> [CLI] schemas update main.test-schema-[UNIQUE_NAME] --new-name test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" +} + +=== With the config unchanged, the plan creates the schema and the volume again under their configured names +>>> [CLI] bundle plan +create schemas.schema1 +create volumes.volume1 + +Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged + +=== With the config renamed to match, the plan finds both at their new names, with the same schema_id and volume_id +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "id": "main.test-schema-renamed-[UNIQUE_NAME]", + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "test-schema-renamed-[UNIQUE_NAME]" + } + }, + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "name": "test-schema-renamed-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[SCHEMA_ID]" + }, + "changes": { + "name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-renamed-[UNIQUE_NAME]" + } + } + }, + "resources.volumes.volume1": { + "id": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "main", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "schema_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-renamed-[UNIQUE_NAME]" + } + } + } +} + +=== Deploy only records the new names in the state +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Updated volumes.volume1 +Files: 3 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog + +>>> [CLI] schemas get main.test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" +} + +>>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "volume_id": "[VOLUME_ID]" +} + +>>> [CLI] bundle summary --output json +{ + "schemas": { + "schema1": "main.test-schema-renamed-[UNIQUE_NAME]" + }, + "volumes": { + "volume1": "main.test-schema-renamed-[UNIQUE_NAME].myvolume" + } +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/script b/acceptance/bundle/resources/schemas/rename_out_of_band/script new file mode 100644 index 00000000000..48f43a5ebc4 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/script @@ -0,0 +1,44 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_SCHEMA="main.test-schema-${UNIQUE_NAME}" +NEW_SCHEMA="main.test-schema-renamed-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +} + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "$OLD_SCHEMA" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "$OLD_SCHEMA.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the schema out of band. UC moves the volume with it" +trace $CLI schemas update "$OLD_SCHEMA" --new-name "test-schema-renamed-${UNIQUE_NAME}" | jq '{full_name, schema_id}' +print_requests.py //unity-catalog &> LOG.rename_requests + +title "With the config unchanged, the plan creates the schema and the volume again under their configured names" +trace $CLI bundle plan + +title "With the config renamed to match, the plan finds both at their new names, with the same schema_id and volume_id" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +plan + +title "Deploy only records the new names in the state" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, schema_id}' +trace $CLI volumes read "$NEW_SCHEMA.myvolume" | jq '{full_name, volume_id}' +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/test.toml b/acceptance/bundle/resources/schemas/rename_out_of_band/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/volumes/recreate_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..2234fdf2e39 --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/out.test.toml b/acceptance/bundle/resources/volumes/recreate_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt b/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt new file mode 100644 index 00000000000..8ebe0b53337 --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt @@ -0,0 +1,81 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Out of band, delete the volume and create another one under a new name +>>> [CLI] volumes delete main.test-schema-[UNIQUE_NAME].myvolume + +=== With the config renamed to match, the plan finds a volume at the new name, but its volume_id differs, so it plans a create +>>> update_file.py databricks.yml name: myvolume name: myvolume2 + +>>> print_state.py +{ + "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", + "stable_id": "[VOLUME_ID]" +} + +>>> [CLI] volumes read main.test-schema-[UNIQUE_NAME].myvolume2 +{ + "full_name": "main.test-schema-[UNIQUE_NAME].myvolume2", + "volume_id": "[NEW_VOLUME_ID]" +} + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS], + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-[UNIQUE_NAME]", + "name": "test-schema-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume2", + "schema_name": "test-schema-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "myvolume", + "new": "myvolume2" + } + } + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/script b/acceptance/bundle/resources/volumes/recreate_out_of_band/script new file mode 100644 index 00000000000..ce1a0dacf83 --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/script @@ -0,0 +1,34 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +SCHEMA="main.test-schema-${UNIQUE_NAME}" + +cleanup() { + # Deleting the schema deletes the volumes in it, the one created out of band too. + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +} + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI volumes read "$SCHEMA.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Out of band, delete the volume and create another one under a new name" +trace $CLI volumes delete "$SCHEMA.myvolume" +NEW_VOLUME_ID=$($CLI volumes create main "test-schema-${UNIQUE_NAME}" myvolume2 MANAGED | jq -r .volume_id) +add_repl "$NEW_VOLUME_ID" NEW_VOLUME_ID +print_requests.py //unity-catalog &> LOG.recreate_requests + +title "With the config renamed to match, the plan finds a volume at the new name, but its volume_id differs, so it plans a create" +trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" +trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, stable_id}' +trace $CLI volumes read "$SCHEMA.myvolume2" | jq '{full_name, volume_id}' +plan diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/test.toml b/acceptance/bundle/resources/volumes/recreate_out_of_band/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/volumes/rename_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..2234fdf2e39 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/out.test.toml b/acceptance/bundle/resources/volumes/rename_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt b/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt new file mode 100644 index 00000000000..80687ae93cc --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt @@ -0,0 +1,121 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the volume out of band +>>> [CLI] volumes update main.test-schema-[UNIQUE_NAME].myvolume --new-name myvolume2 +{ + "full_name": "main.test-schema-[UNIQUE_NAME].myvolume2", + "volume_id": "[VOLUME_ID]" +} + +=== With the config unchanged, the plan creates the volume again under its configured name +>>> [CLI] bundle plan +create volumes.volume1 + +Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged + +=== With the config renamed to match, the plan finds the volume at its new name, with the same volume_id +>>> update_file.py databricks.yml name: myvolume name: myvolume2 + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-[UNIQUE_NAME]", + "name": "test-schema-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + } + }, + "resources.volumes.volume1": { + "id": "main.test-schema-[UNIQUE_NAME].myvolume2", + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume2", + "schema_name": "test-schema-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "main", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "main.test-schema-[UNIQUE_NAME].myvolume2", + "name": "myvolume2", + "owner": "[USERNAME]", + "schema_name": "test-schema-[UNIQUE_NAME]", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "name": { + "action": "skip", + "reason": "remote_already_set", + "old": "myvolume", + "new": "myvolume2", + "remote": "myvolume2" + } + } + } +} + +=== Deploy only records the new name in the state +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog + +>>> [CLI] volumes read main.test-schema-[UNIQUE_NAME].myvolume2 +{ + "full_name": "main.test-schema-[UNIQUE_NAME].myvolume2", + "volume_id": "[VOLUME_ID]" +} + +>>> [CLI] bundle summary --output json +"main.test-schema-[UNIQUE_NAME].myvolume2" + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/script b/acceptance/bundle/resources/volumes/rename_out_of_band/script new file mode 100644 index 00000000000..c16fde22dbc --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/script @@ -0,0 +1,41 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +SCHEMA="main.test-schema-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +} + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI volumes read "$SCHEMA.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the volume out of band" +trace $CLI volumes update "$SCHEMA.myvolume" --new-name myvolume2 | jq '{full_name, volume_id}' +print_requests.py //unity-catalog &> LOG.rename_requests + +title "With the config unchanged, the plan creates the volume again under its configured name" +trace $CLI bundle plan + +title "With the config renamed to match, the plan finds the volume at its new name, with the same volume_id" +trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" +plan + +title "Deploy only records the new name in the state" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI volumes read "$SCHEMA.myvolume2" | jq '{full_name, volume_id}' +trace $CLI bundle summary --output json | jq '.resources.volumes.volume1.id' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/test.toml b/acceptance/bundle/resources/volumes/rename_out_of_band/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] From ae3102e3269a949895e996dcd127c9b971396212 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 6 Nov 2026 13:25:19 +0100 Subject: [PATCH 19/75] acc: normalize a killed CLI on Windows in the interrupted rename tests Co-authored-by: Isaac --- .../bundle/resources/schemas/recreate_interrupted/test.toml | 6 ++++++ .../bundle/resources/schemas/rename_interrupted/test.toml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/test.toml b/acceptance/bundle/resources/schemas/recreate_interrupted/test.toml index 23d6ce8f7fa..e149a3f7289 100644 --- a/acceptance/bundle/resources/schemas/recreate_interrupted/test.toml +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/test.toml @@ -28,3 +28,9 @@ New = 'Exit code: [KILLED]' [[Repls]] Old = '(\[PROCESS_KILLED\]\n\nExit code: )1' New = '${1}[KILLED]' + +# On Windows a killed CLI leaves no bash "Killed" line. The kill lands right after the +# approval text, so its last line is followed directly by "Exit code: 1". +[[Repls]] +Old = '( recreate resources\.volumes\.volume1)\n\nExit code: 1' +New = "${1}\n[PROCESS_KILLED]\n\nExit code: [KILLED]" diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/test.toml b/acceptance/bundle/resources/schemas/rename_interrupted/test.toml index 284671b7e15..443e541f555 100644 --- a/acceptance/bundle/resources/schemas/rename_interrupted/test.toml +++ b/acceptance/bundle/resources/schemas/rename_interrupted/test.toml @@ -27,3 +27,9 @@ New = 'Exit code: [KILLED]' [[Repls]] Old = '(\[PROCESS_KILLED\]\n\nExit code: )1' New = '${1}[KILLED]' + +# On Windows a killed CLI leaves no bash "Killed" line. The kill lands right after the +# "Uploading bundle files" line, so it is followed directly by "Exit code: 1". +[[Repls]] +Old = '(Uploading bundle files to [^\n]+)\n\nExit code: 1' +New = "${1}\n[PROCESS_KILLED]\n\nExit code: [KILLED]" From 17d5e26434c971975ad1b89a967dd690381d74a1 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 6 Nov 2026 15:47:19 +0100 Subject: [PATCH 20/75] direct: record volume_id and schema_id in the resource state instead of stable_id The internal ID now lives in the volume and schema state (VolumeState, SchemaState), set from the API responses and copied by RemapState; the framework reads it back through a new optional InternalID method. This replaces the stable_id field on the state entry and in the DMS envelope. A new drop_changes lifecycle rule leaves the always-different volume_id and schema_id (config never sets them) out of the plan. Co-authored-by: Isaac --- acceptance/bin/print_state.py | 5 +- .../bundle/refschema/output/schemas.txt | 2 +- .../bundle/refschema/output/volumes.txt | 2 +- .../non_existent_field/out.deploy.txt | 2 +- .../non_existent_field/out.plan.txt | 2 +- .../schemas/recreate_out_of_band/output.txt | 16 ++--- .../schemas/recreate_out_of_band/script | 3 +- .../schemas/rename_interrupted/output.txt | 2 +- .../schemas/rename_interrupted/script | 2 +- .../databricks.yml.tmpl | 2 + .../schemas/rename_with_volume_ref/output.txt | 8 +++ .../schemas/rename_with_volume_ref/script | 1 + .../volumes/recreate_out_of_band/output.txt | 2 +- .../volumes/recreate_out_of_band/script | 2 +- bundle/direct/apply.go | 21 ++---- bundle/direct/bundle_plan.go | 32 +++++++--- bundle/direct/bundle_plan_test.go | 19 +++++- bundle/direct/dresources/README.md | 5 +- bundle/direct/dresources/adapter.go | 46 ++++++------- bundle/direct/dresources/config.go | 5 ++ bundle/direct/dresources/config_test.go | 1 + bundle/direct/dresources/configs/schemas.yml | 4 ++ bundle/direct/dresources/configs/volumes.yml | 4 ++ bundle/direct/dresources/schema.go | 64 ++++++++++++++----- bundle/direct/dresources/schema_test.go | 4 +- bundle/direct/dresources/volume.go | 55 ++++++++++++---- bundle/direct/dstate/dms.go | 2 - bundle/direct/dstate/dms_test.go | 13 ---- bundle/direct/dstate/state.go | 17 +---- bundle/direct/dstate/state_test.go | 17 ----- 30 files changed, 209 insertions(+), 151 deletions(-) diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index e7f69ccda56..9badc1e4d5e 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -146,7 +146,6 @@ def get_resources(target): "id": resource.get("resource_id"), "state": envelope.get("state") or {}, "depends_on": envelope.get("depends_on") or [], - "stable_id": envelope.get("stable_id"), } page_token = listed.get("next_page_token") if not page_token: @@ -155,14 +154,12 @@ def get_resources(target): def get_recorded_state(target): """The recorded resources in the on-disk state file's `state` shape (resources. -> - {__id__, state, depends_on, stable_id}), so a recording run prints the same shape as a non-recording one.""" + {__id__, state, depends_on}), so a recording run prints the same shape as a non-recording one.""" state = {} for key, value in sorted(get_resources(target).items()): entry = {"__id__": value["id"], "state": value["state"]} if value["depends_on"]: entry["depends_on"] = value["depends_on"] - if value["stable_id"]: - entry["stable_id"] = value["stable_id"] state[f"resources.{key}"] = entry return state diff --git a/acceptance/bundle/refschema/output/schemas.txt b/acceptance/bundle/refschema/output/schemas.txt index 694d59c6211..566999fd664 100644 --- a/acceptance/bundle/refschema/output/schemas.txt +++ b/acceptance/bundle/refschema/output/schemas.txt @@ -23,7 +23,7 @@ resources.schemas.*.name string ALL resources.schemas.*.owner string REMOTE resources.schemas.*.properties map[string]string ALL resources.schemas.*.properties.* string ALL -resources.schemas.*.schema_id string REMOTE +resources.schemas.*.schema_id string REMOTE STATE resources.schemas.*.storage_location string REMOTE resources.schemas.*.storage_root string ALL resources.schemas.*.updated_at int64 REMOTE diff --git a/acceptance/bundle/refschema/output/volumes.txt b/acceptance/bundle/refschema/output/volumes.txt index 1c573ed1931..52e07594399 100644 --- a/acceptance/bundle/refschema/output/volumes.txt +++ b/acceptance/bundle/refschema/output/volumes.txt @@ -24,7 +24,7 @@ resources.volumes.*.storage_location string ALL resources.volumes.*.updated_at int64 REMOTE resources.volumes.*.updated_by string REMOTE resources.volumes.*.url string INPUT -resources.volumes.*.volume_id string REMOTE +resources.volumes.*.volume_id string REMOTE STATE resources.volumes.*.volume_path string INPUT resources.volumes.*.volume_type catalog.VolumeType ALL resources.volumes.*.grants.full_name string ALL diff --git a/acceptance/bundle/resource_deps/non_existent_field/out.deploy.txt b/acceptance/bundle/resource_deps/non_existent_field/out.deploy.txt index 8da224f9751..d546194dba3 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/out.deploy.txt +++ b/acceptance/bundle/resource_deps/non_existent_field/out.deploy.txt @@ -1,5 +1,5 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in catalog.CreateVolumeRequestContent; non_existent: field "non_existent" not found in catalog.VolumeInfo +Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in dresources.VolumeState; non_existent: field "non_existent" not found in catalog.VolumeInfo Error: planning failed diff --git a/acceptance/bundle/resource_deps/non_existent_field/out.plan.txt b/acceptance/bundle/resource_deps/non_existent_field/out.plan.txt index 6a2b5d1157c..232d9f0b3d4 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/out.plan.txt +++ b/acceptance/bundle/resource_deps/non_existent_field/out.plan.txt @@ -1,4 +1,4 @@ -Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in catalog.CreateVolumeRequestContent; non_existent: field "non_existent" not found in catalog.VolumeInfo +Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in dresources.VolumeState; non_existent: field "non_existent" not found in catalog.VolumeInfo Error: planning failed diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt b/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt index f2520fb4368..f85cf6c5f62 100644 --- a/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt @@ -15,14 +15,14 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> print_state.py { - "resources.schemas.schema1": { - "__id__": "main.test-schema-[UNIQUE_NAME]", - "stable_id": "[SCHEMA_ID]" - }, - "resources.volumes.volume1": { - "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", - "stable_id": "[VOLUME_ID]" - } + "__id__": "main.test-schema-[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" +} + +>>> print_state.py +{ + "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", + "volume_id": "[VOLUME_ID]" } >>> [CLI] schemas get main.test-schema-renamed-[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/script b/acceptance/bundle/resources/schemas/recreate_out_of_band/script index 2d0997ea594..cadca8544af 100644 --- a/acceptance/bundle/resources/schemas/recreate_out_of_band/script +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/script @@ -32,7 +32,8 @@ print_requests.py //unity-catalog &> LOG.recreate_requests title "With the config renamed to match, the plan finds a schema and a volume at the new names, but their ids differ, so it plans creates" trace update_file.py databricks.yml test-schema- test-schema-renamed- -trace print_state.py | jq '.state | map_values({__id__, stable_id})' +trace print_state.py | jq '.state["resources.schemas.schema1"] | {__id__, schema_id: .state.schema_id}' +trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, volume_id: .state.volume_id}' trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, schema_id}' trace $CLI volumes read "$NEW_SCHEMA.myvolume" | jq '{full_name, volume_id}' plan diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt index b2a40eebbfe..f2b97b7ae27 100644 --- a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt +++ b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt @@ -33,7 +33,7 @@ Exit code: [KILLED] >>> print_state.py { "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", - "stable_id": "[VOLUME_ID]" + "volume_id": "[VOLUME_ID]" } >>> [CLI] bundle plan --output json diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/script b/acceptance/bundle/resources/schemas/rename_interrupted/script index 196ee5d5faa..76ee184840a 100644 --- a/acceptance/bundle/resources/schemas/rename_interrupted/script +++ b/acceptance/bundle/resources/schemas/rename_interrupted/script @@ -24,7 +24,7 @@ trace errcode $CLI bundle deploy -q trace print_requests.py //unity-catalog title "The volume is gone from its recorded path, so the plan looks for it at its path in config" -trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, stable_id}' +trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, volume_id: .state.volume_id}' trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"]' title "The next deploy updates the volume at its new path" diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl index 2624f14c7cf..bddda4baff0 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl @@ -19,3 +19,5 @@ resources: default: ${resources.volumes.volume1.full_name} - name: volume_id default: ${resources.volumes.volume1.id} + - name: volume_uuid + default: ${resources.volumes.volume1.volume_id} diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt index 47697e156fd..ceb77a34e65 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt @@ -17,6 +17,10 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged { "default": "main.test-schema-[UNIQUE_NAME].myvolume", "name": "volume_id" + }, + { + "default": "[VOLUME_ID]", + "name": "volume_uuid" } ] @@ -60,6 +64,10 @@ Resources: 0 created, 3 changed, 0 deleted, 0 unchanged { "default": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", "name": "volume_id" + }, + { + "default": "[VOLUME_ID]", + "name": "volume_uuid" } ] diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/script b/acceptance/bundle/resources/schemas/rename_with_volume_ref/script index 0b1627161b3..6418f29011d 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume_ref/script +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/script @@ -13,6 +13,7 @@ trace $CLI bundle deploy print_requests.py //unity-catalog &> LOG.deploy_requests JOB_ID=$($CLI bundle summary --output json | jq -r .resources.jobs.job1.id) add_repl "$JOB_ID" JOB_ID +add_repl "$($CLI volumes read "main.test-schema-${UNIQUE_NAME}.myvolume" | jq -r .volume_id)" VOLUME_ID trace $CLI jobs get "$JOB_ID" | jq .settings.parameters title "Rename the schema" diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt b/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt index 8ebe0b53337..22819d47ef8 100644 --- a/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt @@ -16,7 +16,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> print_state.py { "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", - "stable_id": "[VOLUME_ID]" + "volume_id": "[VOLUME_ID]" } >>> [CLI] volumes read main.test-schema-[UNIQUE_NAME].myvolume2 diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/script b/acceptance/bundle/resources/volumes/recreate_out_of_band/script index ce1a0dacf83..eaaa5d0268a 100644 --- a/acceptance/bundle/resources/volumes/recreate_out_of_band/script +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/script @@ -29,6 +29,6 @@ print_requests.py //unity-catalog &> LOG.recreate_requests title "With the config renamed to match, the plan finds a volume at the new name, but its volume_id differs, so it plans a create" trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" -trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, stable_id}' +trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, volume_id: .state.volume_id}' trace $CLI volumes read "$SCHEMA.myvolume2" | jq '{full_name, volume_id}' plan diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index e1e718d02b4..f47f6c3cdb6 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -93,7 +93,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn, d.Adapter.StableID(d.RemoteState)) + err = d.saveState(ctx, db, newID, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -191,7 +191,7 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("deleting state id=%s: %w", id, err) } } else { - err = d.saveState(ctx, db, id, newState, d.DependsOn, d.stableID(planEntry.RemoteState)) + err = d.saveState(ctx, db, id, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -240,7 +240,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn, d.stableID(planEntry.RemoteState)) + err = d.saveState(ctx, db, newID, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -333,7 +333,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = d.saveState(ctx, db, id, newState, d.DependsOn, d.stableID(entry.RemoteState)) + err = d.saveState(ctx, db, id, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -345,7 +345,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, // with content hashes, see dresources.CompactState) before persisting it. Fields already // smaller than a hash placeholder are persisted as is. Sensitive fields are replaced by a // placeholder value so secrets are never written to disk in plaintext. -func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry, stableID string) error { +func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { compacted, err := dresources.CompactState(d.Adapter.ResourceConfig(), state) if err != nil { return fmt.Errorf("compacting state: %w", err) @@ -353,16 +353,7 @@ func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentSta if err := zeroSensitiveFields(d.Adapter, compacted); err != nil { return fmt.Errorf("redacting state: %w", err) } - return db.SaveStateWithStableID(ctx, d.ResourceKey, newID, compacted, dependsOn, stableID) -} - -// stableID returns the stable ID (see dstate.ResourceEntry) of the object being saved: from the -// remote state the operation returned, else from planRemote, the same object as the plan read it. -func (d *DeploymentUnit) stableID(planRemote any) string { - if id := d.Adapter.StableID(d.RemoteState); id != "" { - return id - } - return d.Adapter.StableID(planRemote) + return db.SaveState(ctx, d.ResourceKey, newID, compacted, dependsOn) } func parseState(destType reflect.Type, raw json.RawMessage) (any, error) { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 5eeaaf86570..f9f225b1e8b 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -335,7 +335,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks } } if remoteState == nil { - entry.ID, remoteState, err = findMoved(ctx, adapter, sv, dbentry) + entry.ID, remoteState, err = findMoved(ctx, adapter, sv, dbentry.ID, savedState) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false @@ -501,6 +501,13 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change return err } + if cfg != nil { + if _, ok := findMatchingRule(path, cfg.DropChanges); ok { + toDrop = append(toDrop, pathString) + continue + } + } + // RemoteAlreadySet only holds when ch.Remote is a real remote value we can compare // against ch.New. Skip it for fields whose remote value is fabricated and thus // meaningless: declared ignore_remote_changes (present in RemoteType but read back @@ -742,15 +749,19 @@ func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan. // findMoved looks for a resource missing from its recorded ID at the ID its config gives it, where a // rename can have moved it: the resource or its parent renamed out of band, or by a deploy that -// stopped before re-keying it. What is found there is taken only if its stable ID matches the -// recorded one, so an unrelated object with the same name is never adopted; the create then fails +// stopped before re-keying it. What is found there is taken only if its internal ID matches the one +// in savedState, so an unrelated object with the same name is never adopted; the create then fails // on it instead. -func findMoved(ctx context.Context, adapter *dresources.Adapter, sv *structvar.StructVar, dbentry dstate.ResourceEntry) (string, any, error) { - if !adapter.HasStableID() || !adapter.HasDoUpdateWithID() || dbentry.StableID == "" || hasUnresolvedIDField(adapter, sv) { +func findMoved(ctx context.Context, adapter *dresources.Adapter, sv *structvar.StructVar, storedID string, savedState any) (string, any, error) { + if !adapter.HasInternalID() || !adapter.HasDoUpdateWithID() || hasUnresolvedIDField(adapter, sv) { return "", nil, nil } + savedInternalID, err := adapter.InternalID(savedState) + if err != nil || savedInternalID == "" { + return "", nil, err + } id, err := adapter.IDFromState(sv.Value) - if err != nil || id == dbentry.ID { + if err != nil || id == storedID { return "", nil, err } remote, err := retry.With(ctx, retry.IsTransientRead, func() (any, error) { @@ -762,8 +773,13 @@ func findMoved(ctx context.Context, adapter *dresources.Adapter, sv *structvar.S if err != nil { return "", nil, fmt.Errorf("reading id=%q: %w", id, err) } - if adapter.StableID(remote) != dbentry.StableID { - return "", nil, nil + found, err := adapter.RemapState(remote) + if err != nil { + return "", nil, err + } + foundInternalID, err := adapter.InternalID(found) + if err != nil || foundInternalID != savedInternalID { + return "", nil, err } return id, remote, nil } diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index f50de986918..1780a13134e 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -55,8 +55,8 @@ func TestDynPathToStructPath(t *testing.T) { func TestExtractReferences_ExcludesReadonlyFields(t *testing.T) { adapters, err := dresources.InitAll(nil) require.NoError(t, err) - // The volume state type is catalog.CreateVolumeRequestContent, which has - // comment but not volume_path. + // The volume state type is VolumeState, which has comment but not + // volume_path. stateType := adapters["volumes"].StateType() const yml = ` @@ -853,3 +853,18 @@ func TestLadderBackendDefaultBeforeRemoteAddition(t *testing.T) { assert.Equal(t, deployplan.Skip, changes["enable_elastic_disk"].Action) assert.Equal(t, deployplan.ReasonBackendDefault, changes["enable_elastic_disk"].Reason) } + +// TestDropChanges: a drop_changes field (volumes' volume_id, which config never sets) leaves the +// plan entirely, while the volume's other changes stay. +func TestDropChanges(t *testing.T) { + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) + + changes := deployplan.Changes{ + "volume_id": {Old: "uuid-1", New: nil, Remote: "uuid-1"}, + "comment": {Old: "a", New: "b", Remote: "a"}, + } + require.NoError(t, addPerFieldActions(t.Context(), adapters["volumes"], changes, nil, nil)) + assert.NotContains(t, changes, "volume_id") + assert.Equal(t, deployplan.Update, changes["comment"].Action) +} diff --git a/bundle/direct/dresources/README.md b/bundle/direct/dresources/README.md index 98f10f85228..b29498ca69b 100644 --- a/bundle/direct/dresources/README.md +++ b/bundle/direct/dresources/README.md @@ -27,6 +27,7 @@ Choose the right category: - `output_only` — the field is computed by the backend; the user never sets it - `input_only` — accepted on create/update but not returned by GET (e.g., write-only tokens, flags) - `managed` — managed by the cloud provider or platform, not by the user config + - **`drop_changes`**: Leave changes to this field out of the plan entirely. Use for fields that config never sets but the state records, like an internal ID recorded from the backend (see `InternalID`). - **`ignore_local_changes`**: Ignore changes the user makes to this field. Use for fields that cannot be updated via API — either they are immutable after creation or require a separate API that is not yet implemented. Must have a comment explaining why. - **`recreate_on_changes`**: Changing this field requires delete + create. Use for truly immutable fields (name, type, location). The reason should reference API docs or TF provider. - **`updatable_id_fields`**: Changing this field changes the resource's ID. Requires `DoUpdateWithID` to be implemented. @@ -47,9 +48,9 @@ For resources whose create or update is asynchronous (the resource is not immedi If the API may return a slice's elements in a different order between calls (e.g., `depends_on` in job tasks, `privileges` in grants), implement `KeyedSlices` to compare elements by a natural key rather than by index. Without this, every deploy after any reordering shows phantom diffs. -## Moved resources: IDFromState / StableID +## Moved resources: IDFromState / InternalID -If the ID is built from the resource's own fields (a UC full name) and the backend has an ID that a rename keeps (a volume's `volume_id`), implement `IDFromState` and `StableID` together, along with `DoUpdateWithID`. When nothing is at the recorded ID, the plan reads at the ID the config gives the resource and takes what it finds only if its stable ID matches the one saved in state. This covers a rename done out of band, or by a deploy that stopped before re-keying the resource. +If the ID is built from the resource's own fields (a UC full name) and the backend has its own ID that a rename keeps (a volume's `volume_id`), record that ID in the state type, set it from the API responses (and from `RemapState`), and implement `IDFromState` and `InternalID` together, along with `DoUpdateWithID`. When nothing is at the recorded ID, the plan reads at the ID the config gives the resource and takes what it finds only if its internal ID matches the one saved in state. This covers a rename done out of band, or by a deploy that stopped before re-keying the resource. ## Empty states: IsEmptyState diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 75c91cdc4d5..64a32906462 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -120,15 +120,15 @@ type IResource interface { IsGone(remoteState any) bool // [Optional] IDFromState returns the ID the state gives the resource, for resource types whose ID - // is built from their own fields (a UC full name). Implemented together with StableID. - // Example: func (*ResourceVolume) IDFromState(state *catalog.CreateVolumeRequestContent) string + // is built from their own fields (a UC full name). Implemented together with InternalID. + // Example: func (*ResourceVolume) IDFromState(state *VolumeState) string IDFromState(state any) string - // [Optional] StableID returns the backend's ID for the object that a rename keeps (a volume's - // volume_id). When the resource is not at its recorded ID, the plan looks for it at the ID - // IDFromState gives it, and takes what it finds only if StableID matches the recorded one. - // Example: func (*ResourceVolume) StableID(remote *catalog.VolumeInfo) string - StableID(remoteState any) string + // [Optional] InternalID returns the backend's own ID recorded in the state, which a rename keeps + // (a volume's volume_id). When the resource is not at its recorded ID, the plan looks for it at the + // ID IDFromState gives it, and takes what it finds only if the internal IDs match. + // Example: func (*ResourceVolume) InternalID(state *VolumeState) string + InternalID(state any) string } // Adapter wraps resource implementation, validates signatures and type consistency across methods @@ -153,7 +153,7 @@ type Adapter struct { doResize *calladapt.BoundCaller isGone *calladapt.BoundCaller idFromState *calladapt.BoundCaller - stableID *calladapt.BoundCaller + internalID *calladapt.BoundCaller resourceConfig *ResourceLifecycleConfig generatedResourceConfig *ResourceLifecycleConfig @@ -196,7 +196,7 @@ func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceC overrideChangeDesc: nil, isGone: nil, idFromState: nil, - stableID: nil, + internalID: nil, resourceConfig: GetResourceConfig(resourceType), generatedResourceConfig: GetGeneratedResourceConfig(resourceType), keyedSlices: nil, @@ -328,7 +328,7 @@ func (a *Adapter) initMethods(resource any) error { return err } - a.stableID, err = calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "StableID") + a.internalID, err = calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "InternalID") if err != nil { return err } @@ -427,12 +427,12 @@ func (a *Adapter) validate() error { validations = append(validations, "IsGone remoteState", a.isGone.InTypes[0], remoteType) } - if (a.idFromState == nil) != (a.stableID == nil) { - return errors.New("IDFromState and StableID must be implemented together") + if (a.idFromState == nil) != (a.internalID == nil) { + return errors.New("IDFromState and InternalID must be implemented together") } if a.idFromState != nil { validations = append(validations, "IDFromState state", a.idFromState.InTypes[0], stateType) - validations = append(validations, "StableID remoteState", a.stableID.InTypes[0], remoteType) + validations = append(validations, "InternalID state", a.internalID.InTypes[0], stateType) } if a.doUpdateWithID != nil { @@ -760,9 +760,9 @@ func (a *Adapter) IsGone(remoteState any) bool { return outs[0].(bool) } -// HasStableID reports whether the resource implements IDFromState and StableID. -func (a *Adapter) HasStableID() bool { - return a.stableID != nil +// HasInternalID reports whether the resource implements IDFromState and InternalID. +func (a *Adapter) HasInternalID() bool { + return a.internalID != nil } // IDFromState returns the ID the state gives the resource; see IResource.IDFromState. @@ -774,17 +774,13 @@ func (a *Adapter) IDFromState(state any) (string, error) { return outs[0].(string), nil } -// StableID returns the rename-proof ID of the object in remoteState; see IResource.StableID. -// It is empty for a resource type without one and for a nil remote state. -func (a *Adapter) StableID(remoteState any) string { - if a.stableID == nil || remoteState == nil || reflect.ValueOf(remoteState).IsNil() { - return "" - } - outs, err := a.stableID.Call(remoteState) +// InternalID returns the backend's own ID recorded in state; see IResource.InternalID. +func (a *Adapter) InternalID(state any) (string, error) { + outs, err := a.internalID.Call(state) if err != nil { - return "" + return "", err } - return outs[0].(string) + return outs[0].(string), nil } // prepareCallRequired prepares a call and ensures the method is found. diff --git a/bundle/direct/dresources/config.go b/bundle/direct/dresources/config.go index 9b3ae9080b8..1cf0f41bb0f 100644 --- a/bundle/direct/dresources/config.go +++ b/bundle/direct/dresources/config.go @@ -68,6 +68,10 @@ type ResourceLifecycleConfig struct { // IgnoreLocalChanges: field patterns where local changes are ignored (can't be updated via API). IgnoreLocalChanges []FieldRule `yaml:"ignore_local_changes,omitempty"` + // DropChanges: field patterns whose changes are left out of the plan: fields that config never + // sets but the state records, such as an internal ID recorded from the backend. + DropChanges []FieldRule `yaml:"drop_changes,omitempty"` + // RecreateOnChanges: field patterns that trigger delete + create when changed. RecreateOnChanges []FieldRule `yaml:"recreate_on_changes,omitempty"` @@ -155,6 +159,7 @@ const ( var empty = ResourceLifecycleConfig{ IgnoreRemoteChanges: nil, IgnoreLocalChanges: nil, + DropChanges: nil, RecreateOnChanges: nil, ProvidedIDFields: nil, UpdatableIDFields: nil, diff --git a/bundle/direct/dresources/config_test.go b/bundle/direct/dresources/config_test.go index 53279eed6c6..d653d1ec695 100644 --- a/bundle/direct/dresources/config_test.go +++ b/bundle/direct/dresources/config_test.go @@ -91,6 +91,7 @@ func categoryRules(c ResourceLifecycleConfig) []struct { }{ {"ignore_remote_changes", c.IgnoreRemoteChanges}, {"ignore_local_changes", c.IgnoreLocalChanges}, + {"drop_changes", c.DropChanges}, {"recreate_on_changes", c.RecreateOnChanges}, {"provided_id_fields", c.ProvidedIDFields}, {"updatable_id_fields", c.UpdatableIDFields}, diff --git a/bundle/direct/dresources/configs/schemas.yml b/bundle/direct/dresources/configs/schemas.yml index 02f68e4afe9..c50e8d12b62 100644 --- a/bundle/direct/dresources/configs/schemas.yml +++ b/bundle/direct/dresources/configs/schemas.yml @@ -16,3 +16,7 @@ backend_defaults: # Without this, every subsequent plan produces an Update whose payload is empty, # and UC rejects it with "UpdateSchema Nothing to update". - field: properties['unity.catalog.managed.*.defaults.*'] +drop_changes: + # Recorded from the backend (see InternalID); config never sets it. + - field: schema_id + reason: internal_id diff --git a/bundle/direct/dresources/configs/volumes.yml b/bundle/direct/dresources/configs/volumes.yml index d4b68e95875..9a6a9906d5e 100644 --- a/bundle/direct/dresources/configs/volumes.yml +++ b/bundle/direct/dresources/configs/volumes.yml @@ -16,3 +16,7 @@ normalize_slash: # https://github.com/databricks/terraform-provider-databricks/blob/v1.65.1/catalog/resource_volume.go#L25 - field: storage_location reason: uc_strips_trailing_slash +drop_changes: + # Recorded from the backend (see InternalID); config never sets it. + - field: volume_id + reason: internal_id diff --git a/bundle/direct/dresources/schema.go b/bundle/direct/dresources/schema.go index 6e1856aa421..250d44fb975 100644 --- a/bundle/direct/dresources/schema.go +++ b/bundle/direct/dresources/schema.go @@ -2,15 +2,33 @@ package dresources import ( "context" + "fmt" "slices" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) +// SchemaState is the schema's config plus its schema_id. The backend assigns schema_id and a +// rename keeps it, so recording it lets the plan recognize the schema under a new name (see +// InternalID). +type SchemaState struct { + catalog.CreateSchema + SchemaId string `json:"schema_id,omitempty"` +} + +func (s *SchemaState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s SchemaState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + type ResourceSchema struct { client *databricks.WorkspaceClient } @@ -19,12 +37,15 @@ func (*ResourceSchema) New(client *databricks.WorkspaceClient) *ResourceSchema { return &ResourceSchema{client: client} } -func (*ResourceSchema) PrepareState(input *resources.Schema) *catalog.CreateSchema { - return &input.CreateSchema +func (*ResourceSchema) PrepareState(input *resources.Schema) *SchemaState { + return &SchemaState{ + CreateSchema: input.CreateSchema, + SchemaId: "", // Recorded from the backend when the schema is created or updated. + } } -func (*ResourceSchema) RemapState(info *catalog.SchemaInfo) *catalog.CreateSchema { - return &catalog.CreateSchema{ +func (*ResourceSchema) RemapState(info *catalog.SchemaInfo) *SchemaState { + return &SchemaState{ CatalogName: info.CatalogName, Comment: info.Comment, CustomMaxRetentionHours: info.CustomMaxRetentionHours, @@ -32,6 +53,7 @@ func (*ResourceSchema) RemapState(info *catalog.SchemaInfo) *catalog.CreateSchem Properties: info.Properties, StorageRoot: info.StorageRoot, ForceSendFields: utils.FilterFields[catalog.CreateSchema](info.ForceSendFields), + SchemaId: info.SchemaId, } } @@ -39,11 +61,12 @@ func (r *ResourceSchema) DoRead(ctx context.Context, id string) (*catalog.Schema return r.client.Schemas.GetByFullName(ctx, id) } -func (r *ResourceSchema) DoCreate(ctx context.Context, config *catalog.CreateSchema) (string, *catalog.SchemaInfo, error) { - response, err := r.client.Schemas.Create(ctx, *config) +func (r *ResourceSchema) DoCreate(ctx context.Context, config *SchemaState) (string, *catalog.SchemaInfo, error) { + response, err := r.client.Schemas.Create(ctx, config.CreateSchema) if err != nil || response == nil { return "", nil, err } + config.SchemaId = response.SchemaId return response.FullName, response, nil } @@ -78,21 +101,22 @@ func makeUpdateSchema(id, newName string, config *catalog.CreateSchema) catalog. } // DoUpdate updates the schema in place and returns remote state. -func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *catalog.CreateSchema, _ *PlanEntry) (*catalog.SchemaInfo, error) { - response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(id, "", config)) - if err != nil { +func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *SchemaState, _ *PlanEntry) (*catalog.SchemaInfo, error) { + response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(id, "", &config.CreateSchema)) + if err != nil || response == nil { return nil, err } - if response != nil && response.FullName != id { + if response.FullName != id { log.Warnf(ctx, "schemas: response contains unexpected full_name=%#v (expected %#v)", response.FullName, id) } + config.SchemaId = response.SchemaId return response, nil } // DoUpdateWithID renames the schema in place when the name changes; the ID is the full name, so it changes too. -func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateSchema, entry *PlanEntry) (string, *catalog.SchemaInfo, error) { +func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config *SchemaState, entry *PlanEntry) (string, *catalog.SchemaInfo, error) { nameFromID, err := getNameFromID(id) if err != nil { return "", nil, err @@ -109,28 +133,34 @@ func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config * if entry.Changes.OnlyParentRenamed() { // UC already moved the schema with its catalog; only the ID changes. + remote, ok := entry.RemoteState.(*catalog.SchemaInfo) + if !ok { + return "", nil, fmt.Errorf("internal error: unexpected remote state type %T", entry.RemoteState) + } + config.SchemaId = remote.SchemaId return fullName, nil, nil } - response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(fullName, newName, config)) + response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(fullName, newName, &config.CreateSchema)) if err != nil || response == nil { return "", nil, err } + config.SchemaId = response.SchemaId return response.FullName, response, nil } // IDFromState returns the full name, which is the ID. -func (*ResourceSchema) IDFromState(state *catalog.CreateSchema) string { +func (*ResourceSchema) IDFromState(state *SchemaState) string { return state.CatalogName + "." + state.Name } -// StableID returns the schema_id, which a rename keeps. -func (*ResourceSchema) StableID(remote *catalog.SchemaInfo) string { - return remote.SchemaId +// InternalID returns the schema_id. +func (*ResourceSchema) InternalID(state *SchemaState) string { + return state.SchemaId } -func (r *ResourceSchema) DoDelete(ctx context.Context, id string, _ *catalog.CreateSchema) error { +func (r *ResourceSchema) DoDelete(ctx context.Context, id string, _ *SchemaState) error { return r.client.Schemas.Delete(ctx, catalog.DeleteSchemaRequest{ FullName: id, Force: true, diff --git a/bundle/direct/dresources/schema_test.go b/bundle/direct/dresources/schema_test.go index ee6397afc6e..5c486cab1f1 100644 --- a/bundle/direct/dresources/schema_test.go +++ b/bundle/direct/dresources/schema_test.go @@ -4,7 +4,6 @@ import ( "encoding/json" "testing" - "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -15,13 +14,14 @@ func TestResourceSchema_DoUpdate_WithUnsupportedForceSendFields(t *testing.T) { adapter := (*ResourceSchema)(nil).New(client) ctx := t.Context() - config := &catalog.CreateSchema{ + config := &SchemaState{ CatalogName: "main", Name: "test_schema", Comment: "original comment", Properties: map[string]string{"key": "value"}, StorageRoot: "", ForceSendFields: nil, + SchemaId: "", } id, _, err := adapter.DoCreate(ctx, config) diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 06674d45f96..5f9f9ccefca 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -10,9 +10,26 @@ import ( "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) +// VolumeState is the volume's config plus its volume_id. The backend assigns volume_id and a +// rename keeps it, so recording it lets the plan recognize the volume under a new name (see +// InternalID). +type VolumeState struct { + catalog.CreateVolumeRequestContent + VolumeId string `json:"volume_id,omitempty"` +} + +func (s *VolumeState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s VolumeState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + type ResourceVolume struct { client *databricks.WorkspaceClient } @@ -21,12 +38,15 @@ func (*ResourceVolume) New(client *databricks.WorkspaceClient) *ResourceVolume { return &ResourceVolume{client: client} } -func (*ResourceVolume) PrepareState(input *resources.Volume) *catalog.CreateVolumeRequestContent { - return &input.CreateVolumeRequestContent +func (*ResourceVolume) PrepareState(input *resources.Volume) *VolumeState { + return &VolumeState{ + CreateVolumeRequestContent: input.CreateVolumeRequestContent, + VolumeId: "", // Recorded from the backend when the volume is created or updated. + } } -func (*ResourceVolume) RemapState(info *catalog.VolumeInfo) *catalog.CreateVolumeRequestContent { - return &catalog.CreateVolumeRequestContent{ +func (*ResourceVolume) RemapState(info *catalog.VolumeInfo) *VolumeState { + return &VolumeState{ CatalogName: info.CatalogName, Comment: info.Comment, Name: info.Name, @@ -34,6 +54,7 @@ func (*ResourceVolume) RemapState(info *catalog.VolumeInfo) *catalog.CreateVolum StorageLocation: info.StorageLocation, VolumeType: info.VolumeType, ForceSendFields: utils.FilterFields[catalog.CreateVolumeRequestContent](info.ForceSendFields), + VolumeId: info.VolumeId, } } @@ -41,18 +62,19 @@ func (r *ResourceVolume) DoRead(ctx context.Context, id string) (*catalog.Volume return r.client.Volumes.ReadByName(ctx, id) } -func (r *ResourceVolume) DoCreate(ctx context.Context, config *catalog.CreateVolumeRequestContent) (string, *catalog.VolumeInfo, error) { - response, err := r.client.Volumes.Create(ctx, *config) +func (r *ResourceVolume) DoCreate(ctx context.Context, config *VolumeState) (string, *catalog.VolumeInfo, error) { + response, err := r.client.Volumes.Create(ctx, config.CreateVolumeRequestContent) if err != nil { return "", nil, err } + config.VolumeId = response.VolumeId return response.FullName, response, nil } // See schemaForceSend. Verified against a real workspace: {"comment": ""} clears it. var volumeForceSend = []string{"Comment"} -func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *catalog.CreateVolumeRequestContent, _ *PlanEntry) (*catalog.VolumeInfo, error) { +func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *VolumeState, _ *PlanEntry) (*catalog.VolumeInfo, error) { updateRequest := catalog.UpdateVolumeRequestContent{ Comment: config.Comment, Name: id, @@ -80,10 +102,11 @@ func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *catalo log.Warnf(ctx, "volumes: response contains unexpected full_name=%#v (expected %#v)", response.FullName, id) } + config.VolumeId = response.VolumeId return response, err } -func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateVolumeRequestContent, entry *PlanEntry) (string, *catalog.VolumeInfo, error) { +func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config *VolumeState, entry *PlanEntry) (string, *catalog.VolumeInfo, error) { updateRequest := catalog.UpdateVolumeRequestContent{ Comment: config.Comment, Name: "", // Initialized below @@ -106,6 +129,11 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * if entry.Changes.OnlyParentRenamed() { // UC already moved the volume with its parent; only the ID changes. + remote, ok := entry.RemoteState.(*catalog.VolumeInfo) + if !ok { + return "", nil, fmt.Errorf("internal error: unexpected remote state type %T", entry.RemoteState) + } + config.VolumeId = remote.VolumeId return updateRequest.Name, nil, nil } @@ -118,20 +146,21 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * return "", nil, err } + config.VolumeId = response.VolumeId return response.FullName, response, nil } // IDFromState returns the full name, which is the ID. -func (*ResourceVolume) IDFromState(state *catalog.CreateVolumeRequestContent) string { +func (*ResourceVolume) IDFromState(state *VolumeState) string { return state.CatalogName + "." + state.SchemaName + "." + state.Name } -// StableID returns the volume_id, which a rename keeps. -func (*ResourceVolume) StableID(remote *catalog.VolumeInfo) string { - return remote.VolumeId +// InternalID returns the volume_id. +func (*ResourceVolume) InternalID(state *VolumeState) string { + return state.VolumeId } -func (r *ResourceVolume) DoDelete(ctx context.Context, id string, _ *catalog.CreateVolumeRequestContent) error { +func (r *ResourceVolume) DoDelete(ctx context.Context, id string, _ *VolumeState) error { return r.client.Volumes.DeleteByName(ctx, id) } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 4842b8de10b..57e10b8d4b4 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -14,7 +14,6 @@ import ( type RecordedState struct { State json.RawMessage `json:"state"` DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` - StableID string `json:"stable_id,omitempty"` } // applyDMSState fills in the resource state from what DMS recorded. For a deployment that uses DMS @@ -58,6 +57,5 @@ func stateEntry(res dms.Resource) (ResourceEntry, error) { ID: res.ID, State: recorded.State, DependsOn: recorded.DependsOn, - StableID: recorded.StableID, }, nil } diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index 0dc86caf879..ab4e1bc017f 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -44,19 +44,6 @@ func TestApplyDMSState(t *testing.T) { "resources.pipelines.bar": {ID: "456"}, }, }, - { - name: "stable_id comes back from the envelope", - recorded: []dms.Resource{ - {Key: "resources.schemas.foo", ID: "main.foo", State: `{"state":{"name":"foo"},"stable_id":"uuid-1"}`}, - }, - want: map[string]ResourceEntry{ - "resources.schemas.foo": { - ID: "main.foo", - State: json.RawMessage(`{"name":"foo"}`), - StableID: "uuid-1", - }, - }, - }, { // Nil is what a list holding nothing returns, and it means a successful deploy of // nothing rather than missing data. diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index bd8bfc97e23..de00e76233a 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -169,11 +169,6 @@ type ResourceEntry struct { ID string `json:"__id__"` State json.RawMessage `json:"state"` DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` - - // StableID is the backend's ID for the object that survives a rename (a volume's volume_id), - // for resource types that have one. When the object is not at ID any more, the plan looks for - // it at the ID its config gives it and takes what it finds there only if StableID matches. - StableID string `json:"stable_id,omitempty"` } type WALEntry struct { @@ -294,7 +289,7 @@ func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause e var recorded json.RawMessage if entry, ok := db.GetResourceEntry(resourceKey); resourceID != "" && ok && len(entry.State) > 0 { var err error - recorded, err = json.Marshal(RecordedState{State: entry.State, DependsOn: entry.DependsOn, StableID: entry.StableID}) + recorded, err = json.Marshal(RecordedState{State: entry.State, DependsOn: entry.DependsOn}) if err != nil { // Nothing the caller can act on, so record the failure without the state. recorded = nil @@ -320,11 +315,6 @@ func NewDatabase(lineage string, serial int) Database { // SaveState records the resource's state after an operation was applied to it. func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { - return db.SaveStateWithStableID(ctx, key, newID, state, dependsOn, "") -} - -// SaveStateWithStableID is SaveState that also records the object's stable ID (see ResourceEntry). -func (db *DeploymentState) SaveStateWithStableID(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry, stableID string) error { db.AssertOpenedForWrite() db.mu.Lock() defer db.mu.Unlock() @@ -341,7 +331,6 @@ func (db *DeploymentState) SaveStateWithStableID(ctx context.Context, key, newID ID: newID, State: json.RawMessage(jsonMessage), DependsOn: dependsOn, - StableID: stableID, } switch db.mode { @@ -355,7 +344,7 @@ func (db *DeploymentState) SaveStateWithStableID(ctx context.Context, key, newID db.Data.State[key] = entry db.stateIDs[key] = newID if buf := db.operationBuffer; buf != nil { - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, StableID: stableID}) + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) if err != nil { return err } @@ -370,7 +359,7 @@ func (db *DeploymentState) SaveStateWithStableID(ctx context.Context, key, newID } db.stateIDs[key] = newID if buf := db.operationBuffer; buf != nil { - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn, StableID: stableID}) + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) if err != nil { return err } diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 6330a5d99e4..c5bfea2783b 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -38,23 +38,6 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { mustFinalize(t, &db2) } -func TestSaveStateWithStableIDRoundTrip(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.json") - - var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WritingFile, WithRecovery(true), OpenDmsArgs{})) - require.NoError(t, db.SaveStateWithStableID(t.Context(), "schemas.foo", "main.foo", map[string]string{"name": "foo"}, nil, "uuid-1")) - mustFinalize(t, &db) - - var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) - entry, ok := db2.GetResourceEntry("schemas.foo") - require.True(t, ok) - assert.Equal(t, "main.foo", entry.ID) - assert.Equal(t, "uuid-1", entry.StableID) - mustFinalize(t, &db2) -} - func TestFinalizeWithoutResourceWrites(t *testing.T) { for _, mode := range []Mode{WritingFile, WritingDeploymentHistoryShadow} { path := filepath.Join(t.TempDir(), "state.json") From bd7a2ac602e8eda06ad58fd3aff405c7485f3baa Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 11:16:18 +0100 Subject: [PATCH 21/75] Regenerate apitypes and volume spec rules for the new state types With volume_id and schema_id in the state, the field-overlap match picks VolumeInfo and SchemaInfo, as registered models already map to their Info type; volume_id is marked output-only from the spec. Co-authored-by: Isaac --- bundle/direct/dresources/apitypes.generated.yml | 4 ++-- bundle/direct/dresources/configs/volumes.generated.yml | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 bundle/direct/dresources/configs/volumes.generated.yml diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index db57a08bc8d..fb16c02facf 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -72,7 +72,7 @@ quality_monitors: catalog.CreateMonitor registered_models: catalog.RegisteredModelInfo -schemas: catalog.CreateSchema +schemas: catalog.SchemaInfo secret_scopes: workspace.CreateScope @@ -86,4 +86,4 @@ vector_search_endpoints: vectorsearch.CreateEndpoint vector_search_indexes: vectorsearch.CreateVectorIndexRequest -volumes: catalog.CreateVolumeRequestContent +volumes: catalog.VolumeInfo diff --git a/bundle/direct/dresources/configs/volumes.generated.yml b/bundle/direct/dresources/configs/volumes.generated.yml new file mode 100644 index 00000000000..3faf600a2dd --- /dev/null +++ b/bundle/direct/dresources/configs/volumes.generated.yml @@ -0,0 +1,5 @@ +# Generated, do not edit. + +ignore_remote_changes: + - field: volume_id + reason: spec:output_only From e45d36f1f7517b8a466360039dcacd47aa912abd Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 11:35:26 +0100 Subject: [PATCH 22/75] direct: ignore volume_id and schema_id changes instead of dropping them Use ignore_local_changes (config never sets them) and ignore_remote_changes (generated from the spec for volume_id, hand-written for schema_id, which the spec does not mark output-only) instead of a new drop_changes rule. The skipped change keeps the recorded and remote IDs visible in the plan. Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 7 ------ bundle/direct/bundle_plan_test.go | 26 +++++++++++++------- bundle/direct/dresources/README.md | 1 - bundle/direct/dresources/config.go | 5 ---- bundle/direct/dresources/config_test.go | 1 - bundle/direct/dresources/configs/schemas.yml | 7 +++++- bundle/direct/dresources/configs/volumes.yml | 5 ++-- 7 files changed, 26 insertions(+), 26 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index f9f225b1e8b..c3aba60d2ec 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -501,13 +501,6 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change return err } - if cfg != nil { - if _, ok := findMatchingRule(path, cfg.DropChanges); ok { - toDrop = append(toDrop, pathString) - continue - } - } - // RemoteAlreadySet only holds when ch.Remote is a real remote value we can compare // against ch.New. Skip it for fields whose remote value is fabricated and thus // meaningless: declared ignore_remote_changes (present in RemoteType but read back diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 1780a13134e..18050e021f2 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -854,17 +854,25 @@ func TestLadderBackendDefaultBeforeRemoteAddition(t *testing.T) { assert.Equal(t, deployplan.ReasonBackendDefault, changes["enable_elastic_disk"].Reason) } -// TestDropChanges: a drop_changes field (volumes' volume_id, which config never sets) leaves the -// plan entirely, while the volume's other changes stay. -func TestDropChanges(t *testing.T) { +// TestInternalIDChangesAreSkipped: volume_id and schema_id are recorded in state but never set by +// config, so neither the always-different local value nor a state recorded before them plans an update. +func TestInternalIDChangesAreSkipped(t *testing.T) { adapters, err := dresources.InitAll(nil) require.NoError(t, err) - changes := deployplan.Changes{ - "volume_id": {Old: "uuid-1", New: nil, Remote: "uuid-1"}, - "comment": {Old: "a", New: "b", Remote: "a"}, + for _, tt := range []struct{ resource, field string }{ + {"volumes", "volume_id"}, + {"schemas", "schema_id"}, + } { + for name, ch := range map[string]*deployplan.ChangeDesc{ + "recorded": {Old: "uuid-1", New: nil, Remote: "uuid-1"}, + "not recorded": {Old: nil, New: nil, Remote: "uuid-1"}, + } { + t.Run(tt.resource+" "+name, func(t *testing.T) { + changes := deployplan.Changes{tt.field: ch} + require.NoError(t, addPerFieldActions(t.Context(), adapters[tt.resource], changes, nil, nil)) + assert.Equal(t, deployplan.Skip, changes[tt.field].Action) + }) + } } - require.NoError(t, addPerFieldActions(t.Context(), adapters["volumes"], changes, nil, nil)) - assert.NotContains(t, changes, "volume_id") - assert.Equal(t, deployplan.Update, changes["comment"].Action) } diff --git a/bundle/direct/dresources/README.md b/bundle/direct/dresources/README.md index b29498ca69b..c31853fe6d7 100644 --- a/bundle/direct/dresources/README.md +++ b/bundle/direct/dresources/README.md @@ -27,7 +27,6 @@ Choose the right category: - `output_only` — the field is computed by the backend; the user never sets it - `input_only` — accepted on create/update but not returned by GET (e.g., write-only tokens, flags) - `managed` — managed by the cloud provider or platform, not by the user config - - **`drop_changes`**: Leave changes to this field out of the plan entirely. Use for fields that config never sets but the state records, like an internal ID recorded from the backend (see `InternalID`). - **`ignore_local_changes`**: Ignore changes the user makes to this field. Use for fields that cannot be updated via API — either they are immutable after creation or require a separate API that is not yet implemented. Must have a comment explaining why. - **`recreate_on_changes`**: Changing this field requires delete + create. Use for truly immutable fields (name, type, location). The reason should reference API docs or TF provider. - **`updatable_id_fields`**: Changing this field changes the resource's ID. Requires `DoUpdateWithID` to be implemented. diff --git a/bundle/direct/dresources/config.go b/bundle/direct/dresources/config.go index 1cf0f41bb0f..9b3ae9080b8 100644 --- a/bundle/direct/dresources/config.go +++ b/bundle/direct/dresources/config.go @@ -68,10 +68,6 @@ type ResourceLifecycleConfig struct { // IgnoreLocalChanges: field patterns where local changes are ignored (can't be updated via API). IgnoreLocalChanges []FieldRule `yaml:"ignore_local_changes,omitempty"` - // DropChanges: field patterns whose changes are left out of the plan: fields that config never - // sets but the state records, such as an internal ID recorded from the backend. - DropChanges []FieldRule `yaml:"drop_changes,omitempty"` - // RecreateOnChanges: field patterns that trigger delete + create when changed. RecreateOnChanges []FieldRule `yaml:"recreate_on_changes,omitempty"` @@ -159,7 +155,6 @@ const ( var empty = ResourceLifecycleConfig{ IgnoreRemoteChanges: nil, IgnoreLocalChanges: nil, - DropChanges: nil, RecreateOnChanges: nil, ProvidedIDFields: nil, UpdatableIDFields: nil, diff --git a/bundle/direct/dresources/config_test.go b/bundle/direct/dresources/config_test.go index d653d1ec695..53279eed6c6 100644 --- a/bundle/direct/dresources/config_test.go +++ b/bundle/direct/dresources/config_test.go @@ -91,7 +91,6 @@ func categoryRules(c ResourceLifecycleConfig) []struct { }{ {"ignore_remote_changes", c.IgnoreRemoteChanges}, {"ignore_local_changes", c.IgnoreLocalChanges}, - {"drop_changes", c.DropChanges}, {"recreate_on_changes", c.RecreateOnChanges}, {"provided_id_fields", c.ProvidedIDFields}, {"updatable_id_fields", c.UpdatableIDFields}, diff --git a/bundle/direct/dresources/configs/schemas.yml b/bundle/direct/dresources/configs/schemas.yml index c50e8d12b62..6517267b988 100644 --- a/bundle/direct/dresources/configs/schemas.yml +++ b/bundle/direct/dresources/configs/schemas.yml @@ -16,7 +16,12 @@ backend_defaults: # Without this, every subsequent plan produces an Update whose payload is empty, # and UC rejects it with "UpdateSchema Nothing to update". - field: properties['unity.catalog.managed.*.defaults.*'] -drop_changes: +ignore_local_changes: # Recorded from the backend (see InternalID); config never sets it. - field: schema_id reason: internal_id +ignore_remote_changes: + # The spec does not mark schema_id output-only, so no generated rule covers a state + # recorded before schema_id was. + - field: schema_id + reason: internal_id diff --git a/bundle/direct/dresources/configs/volumes.yml b/bundle/direct/dresources/configs/volumes.yml index 9a6a9906d5e..3c733277d18 100644 --- a/bundle/direct/dresources/configs/volumes.yml +++ b/bundle/direct/dresources/configs/volumes.yml @@ -16,7 +16,8 @@ normalize_slash: # https://github.com/databricks/terraform-provider-databricks/blob/v1.65.1/catalog/resource_volume.go#L25 - field: storage_location reason: uc_strips_trailing_slash -drop_changes: - # Recorded from the backend (see InternalID); config never sets it. +ignore_local_changes: + # Recorded from the backend (see InternalID); config never sets it. Remote changes are + # ignored by the generated rule (the spec marks it output-only). - field: volume_id reason: internal_id From 3a86513a33cba6f23c6dab6dbfbdff6ab09d4721 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 12:40:48 +0100 Subject: [PATCH 23/75] acc: show the skipped volume_id and schema_id changes in plans Co-authored-by: Isaac --- .../bundle/resources/catalogs/rename/output.txt | 12 ++++++++++++ .../catalogs/rename_all_out_of_band/output.txt | 12 ++++++++++++ .../catalogs/rename_out_of_band/output.txt | 12 ++++++++++++ .../schemas/recreate_interrupted/output.txt | 5 +++++ .../schemas/recreate_out_of_band/output.txt | 10 ++++++++++ .../bundle/resources/schemas/rename/output.txt | 6 ++++++ .../schemas/rename_interrupted/output.txt | 6 ++++++ .../schemas/rename_out_of_band/output.txt | 12 ++++++++++++ .../schemas/rename_with_volume/output.txt | 12 ++++++++++++ .../volumes/recreate_out_of_band/output.txt | 13 +++++++++++++ .../volumes/rename_out_of_band/output.txt | 14 ++++++++++++++ .../volumes/uppercase-name/out.deploy.txt | 8 ++++++++ 12 files changed, 122 insertions(+) diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index 07843301bcc..d5910306690 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -97,6 +97,12 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "old": "test_catalog_[UNIQUE_NAME]", "new": "test_catalog_renamed_[UNIQUE_NAME]", "remote": "test_catalog_[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]", + "remote": "[SCHEMA_ID]" } } }, @@ -138,6 +144,12 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "old": "test_catalog_[UNIQUE_NAME]", "new": "test_catalog_renamed_[UNIQUE_NAME]", "remote": "test_catalog_[UNIQUE_NAME]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" } } } diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt index 7b1f43ba6ce..c57f61199d1 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt @@ -89,6 +89,12 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "old": "myschema", "new": "myschema2", "remote": "myschema2" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]", + "remote": "[SCHEMA_ID]" } } }, @@ -145,6 +151,12 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "old": "myschema", "new": "myschema2", "remote": "myschema2" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" } } } diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt index 09bf7199065..fccb0c8b17b 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt @@ -68,6 +68,12 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "old": "test_catalog_[UNIQUE_NAME]", "new": "test_catalog_renamed_[UNIQUE_NAME]", "remote": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]", + "remote": "[SCHEMA_ID]" } } }, @@ -110,6 +116,12 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "old": "test_catalog_[UNIQUE_NAME]", "new": "test_catalog_renamed_[UNIQUE_NAME]", "remote": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" } } } diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt b/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt index 69ff54088bd..8d92949d3e0 100644 --- a/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt @@ -69,6 +69,11 @@ Exit code: [KILLED] "reason": "id_field", "old": "test-schema-[UNIQUE_NAME]", "new": "test-schema-renamed-[UNIQUE_NAME]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]" } } } diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt b/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt index f85cf6c5f62..36e38a1c807 100644 --- a/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt @@ -53,6 +53,11 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "reason": "id_changes", "old": "test-schema-[UNIQUE_NAME]", "new": "test-schema-renamed-[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]" } } }, @@ -78,6 +83,11 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "reason": "id_field", "old": "test-schema-[UNIQUE_NAME]", "new": "test-schema-renamed-[UNIQUE_NAME]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]" } } } diff --git a/acceptance/bundle/resources/schemas/rename/output.txt b/acceptance/bundle/resources/schemas/rename/output.txt index 275d84698ac..57cab932c15 100644 --- a/acceptance/bundle/resources/schemas/rename/output.txt +++ b/acceptance/bundle/resources/schemas/rename/output.txt @@ -53,6 +53,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "old": "test-schema-[UNIQUE_NAME]", "new": "test-schema-renamed-[UNIQUE_NAME]", "remote": "test-schema-[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]", + "remote": "[SCHEMA_ID]" } } } diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt index f2b97b7ae27..6c768de425a 100644 --- a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt +++ b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt @@ -87,6 +87,12 @@ Exit code: [KILLED] "action": "skip", "reason": "immutable_backend_value", "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[VOLUME_ID]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" } } } diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt b/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt index 0b951a5e912..afc72aaab04 100644 --- a/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt @@ -54,6 +54,12 @@ Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged "old": "test-schema-[UNIQUE_NAME]", "new": "test-schema-renamed-[UNIQUE_NAME]", "remote": "test-schema-renamed-[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]", + "remote": "[SCHEMA_ID]" } } }, @@ -92,6 +98,12 @@ Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged "old": "test-schema-[UNIQUE_NAME]", "new": "test-schema-renamed-[UNIQUE_NAME]", "remote": "test-schema-renamed-[UNIQUE_NAME]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" } } } diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt index 37458a8f7bf..c49df8092b0 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -60,6 +60,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "old": "test-schema-[UNIQUE_NAME]", "new": "test-schema-renamed-[UNIQUE_NAME]", "remote": "test-schema-[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" } } }, @@ -97,6 +103,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "old": "test-schema-[UNIQUE_NAME]", "new": "test-schema-renamed-[UNIQUE_NAME]", "remote": "test-schema-[UNIQUE_NAME]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" } } } diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt b/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt index 22819d47ef8..b8b36ce7130 100644 --- a/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt @@ -40,6 +40,14 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "name": "test-schema-[UNIQUE_NAME]", "owner": "[USERNAME]", "schema_id": "[UUID]" + }, + "changes": { + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } } }, "resources.volumes.volume1": { @@ -64,6 +72,11 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "reason": "id_changes", "old": "myvolume", "new": "myvolume2" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]" } } } diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt b/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt index 80687ae93cc..d9309f73e35 100644 --- a/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt @@ -38,6 +38,14 @@ Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged "name": "test-schema-[UNIQUE_NAME]", "owner": "[USERNAME]", "schema_id": "[UUID]" + }, + "changes": { + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } } }, "resources.volumes.volume1": { @@ -75,6 +83,12 @@ Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged "old": "myvolume", "new": "myvolume2", "remote": "myvolume2" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" } } } diff --git a/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.txt b/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.txt index 2b50dcb8cfa..f58c45bb68f 100644 --- a/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.txt +++ b/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.txt @@ -34,6 +34,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "name": { "action": "skip", "reason": "id_changes" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id" } } }, @@ -51,6 +55,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "storage_location": { "action": "skip", "reason": "immutable_backend_value" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id" } } } From 1aa7f3f17df32a9bc2018cb30c8b1699bd39c921 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 13:26:57 +0100 Subject: [PATCH 24/75] acc: a volume follows its catalog and schema renamed in one deploy Co-authored-by: Isaac --- .../databricks.yml.tmpl | 16 ++ .../rename_catalog_and_schema/out.test.toml | 2 + .../rename_catalog_and_schema/output.txt | 223 ++++++++++++++++++ .../volumes/rename_catalog_and_schema/script | 37 +++ .../rename_catalog_and_schema/test.toml | 7 + 5 files changed, 285 insertions(+) create mode 100644 acceptance/bundle/resources/volumes/rename_catalog_and_schema/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/volumes/rename_catalog_and_schema/out.test.toml create mode 100644 acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt create mode 100644 acceptance/bundle/resources/volumes/rename_catalog_and_schema/script create mode 100644 acceptance/bundle/resources/volumes/rename_catalog_and_schema/test.toml diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/databricks.yml.tmpl b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/databricks.yml.tmpl new file mode 100644 index 00000000000..e45eccddc44 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + volumes: + volume1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/out.test.toml b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt new file mode 100644 index 00000000000..a0fac10b722 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt @@ -0,0 +1,223 @@ + +=== Deploy a catalog with a schema and a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the catalog and the schema in one deploy +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +>>> update_file.py databricks.yml name: myschema name: myschema2 + +=== Both of the volume's parents are renamed, so both its catalog_name and schema_name follow them +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "update_id", + "new_state": { + "value": { + "name": "test_catalog_renamed_[UNIQUE_NAME]" + } + }, + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME]", + "name": "test_catalog_[UNIQUE_NAME]", + "owner": "[USERNAME]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + } + }, + "resources.schemas.schema1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema", + "name": "myschema", + "owner": "[USERNAME]", + "schema_id": "[SCHEMA_ID]" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]", + "remote": "[SCHEMA_ID]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myvolume", + "schema_name": "myschema2", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "created_at": [UNIX_TIME_MILLIS][2], + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema.myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "myschema", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" + } + } + } +} + +=== Deploy renames the catalog, then the schema under it. UC moves the volume both times, so it gets no request +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated catalogs.catalog1 +Updated schemas.schema1 +Updated volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 3 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test_catalog_renamed_[UNIQUE_NAME]" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/test_catalog_renamed_[UNIQUE_NAME].myschema", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "myschema2" + } +} + +=== The schema and the volume are under the new names, with their original ids +>>> [CLI] schemas get test_catalog_renamed_[UNIQUE_NAME].myschema2 +{ + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2", + "schema_id": "[SCHEMA_ID]" +} + +>>> [CLI] volumes read test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume +{ + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume", + "volume_id": "[VOLUME_ID]" +} + +>>> [CLI] bundle summary --output json +{ + "catalogs": { + "catalog1": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "schemas": { + "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema2" + }, + "volumes": { + "volume1": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume" + } +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/script b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/script new file mode 100644 index 00000000000..7b2ea667360 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/script @@ -0,0 +1,37 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +NEW_SCHEMA="test_catalog_renamed_${UNIQUE_NAME}.myschema2" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a catalog with a schema and a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "test_catalog_${UNIQUE_NAME}.myschema" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "test_catalog_${UNIQUE_NAME}.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the catalog and the schema in one deploy" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ +trace update_file.py databricks.yml "name: myschema" "name: myschema2" + +title "Both of the volume's parents are renamed, so both its catalog_name and schema_name follow them" +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location + +title "Deploy renames the catalog, then the schema under it. UC moves the volume both times, so it gets no request" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "The schema and the volume are under the new names, with their original ids" +trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, schema_id}' +trace $CLI volumes read "$NEW_SCHEMA.myvolume" | jq '{full_name, volume_id}' +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/test.toml b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] From 2b5169d0591c26040edcf2addc029cc1adb4eafc Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 13:31:01 +0100 Subject: [PATCH 25/75] acc: keep the captured migrate fixtures; let the replay create the catalog The captured requests already create the destination catalog out of band, so the scripts no longer create it again. This restores terraform-requests.json as captured instead of editing it. Co-authored-by: Isaac --- .../terraform-requests.json | 7 +++++++ .../auto/schema-catalog-change/terraform-requests.json | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/acceptance/bundle/migrate/auto/registered-model-catalog-change/terraform-requests.json b/acceptance/bundle/migrate/auto/registered-model-catalog-change/terraform-requests.json index 6c27b1fc729..6f4620d81f5 100644 --- a/acceptance/bundle/migrate/auto/registered-model-catalog-change/terraform-requests.json +++ b/acceptance/bundle/migrate/auto/registered-model-catalog-change/terraform-requests.json @@ -1,4 +1,11 @@ [ + { + "method": "POST", + "path": "/api/2.1/unity-catalog/catalogs", + "body": { + "name": "othercat_[UNIQUE_NAME]" + } + }, { "method": "POST", "path": "/api/2.1/unity-catalog/models", diff --git a/acceptance/bundle/migrate/auto/schema-catalog-change/terraform-requests.json b/acceptance/bundle/migrate/auto/schema-catalog-change/terraform-requests.json index e7072606fd3..eda99b84d42 100644 --- a/acceptance/bundle/migrate/auto/schema-catalog-change/terraform-requests.json +++ b/acceptance/bundle/migrate/auto/schema-catalog-change/terraform-requests.json @@ -1,4 +1,11 @@ [ + { + "method": "POST", + "path": "/api/2.1/unity-catalog/catalogs", + "body": { + "name": "othercat_[UNIQUE_NAME]" + } + }, { "method": "POST", "path": "/api/2.1/unity-catalog/schemas", From 6b609e472a7208f25645a4dc977490bd74de323f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 14:05:40 +0100 Subject: [PATCH 26/75] direct: classify a parent-renamed ID field as update_id directly The parent-rename rule now runs where provided ID fields are classified, so the change is planned as update_id the first time instead of being planned as a recreate and rewritten by a separate pass. The plan output is unchanged. Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 44 +++++++++---------------------- bundle/direct/bundle_plan_test.go | 20 +++++++------- 2 files changed, 22 insertions(+), 42 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index c3aba60d2ec..7c1b8981595 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -378,13 +378,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } - err = addPerFieldActions(ctx, adapter, entry.Changes, sv.Value, remoteState) - if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: classifying changes: %w", errorPrefix, err)) - return false - } - - err = followParentRenames(adapter, entry.Changes, parentRenames) + err = addPerFieldActions(ctx, adapter, entry.Changes, sv.Value, remoteState, parentRenames) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: classifying changes: %w", errorPrefix, err)) return false @@ -489,7 +483,7 @@ func prepareChanges(ctx context.Context, adapter *dresources.Adapter, localDiff, return m, nil } -func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, changes deployplan.Changes, newState, remoteState any) error { +func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, changes deployplan.Changes, newState, remoteState any, parentRenames map[string]*deployplan.ChangeDesc) error { cfg := adapter.ResourceConfig() generatedCfg := adapter.GeneratedResourceConfig() @@ -522,6 +516,10 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change } else if action, reason, ok := classifyIDField(cfg, path, ch); ok { ch.Action = action ch.Reason = reason + if action == deployplan.Recreate && followsParentRename(adapter, parentRenames[pathString], ch) { + ch.Action = deployplan.UpdateWithID + ch.Reason = deployplan.ReasonParentRenamed + } } else if action, reason, ok := classifyIDField(generatedCfg, path, ch); ok { ch.Action = action ch.Reason = reason @@ -795,33 +793,15 @@ func hasUnresolvedIDField(adapter *dresources.Adapter, sv *structvar.StructVar) return false } -// followParentRenames plans an update_id instead of a recreate for an ID field that changes -// only because it references the name of a parent that is renamed in place. UC moves the -// children of a renamed catalog or schema along with it, so recreating such a child fails -// with "already exists"; its DoUpdateWithID addresses it under the new parent instead. +// followsParentRename reports whether a change to a provided ID field only follows parentChange, +// the rename of the parent field it references (see parentRenames). UC moves the children of a +// renamed catalog or schema along with it, so recreating such a child fails with "already +// exists"; it is planned as an update_id instead, which addresses it under the new parent. // // The old values must match too: a child moved to the renamed parent from somewhere else // still has to be recreated. -func followParentRenames(adapter *dresources.Adapter, changes deployplan.Changes, parentRenames map[string]*deployplan.ChangeDesc) error { - if !adapter.HasDoUpdateWithID() { - return nil - } - for pathString, parentChange := range parentRenames { - ch := changes[pathString] - if ch == nil || ch.Action != deployplan.Recreate || !structdiff.IsEqual(ch.Old, parentChange.Old) { - continue - } - path, err := structpath.ParsePath(pathString) - if err != nil { - return err - } - if _, ok := findMatchingRule(path, adapter.ResourceConfig().ProvidedIDFields); !ok { - continue - } - ch.Action = deployplan.UpdateWithID - ch.Reason = deployplan.ReasonParentRenamed - } - return nil +func followsParentRename(adapter *dresources.Adapter, parentChange, ch *deployplan.ChangeDesc) bool { + return parentChange != nil && adapter.HasDoUpdateWithID() && structdiff.IsEqual(ch.Old, parentChange.Old) } // shouldSkipNormalized skips a change that is a false diff caused by UC API diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 18050e021f2..42101800f21 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -259,7 +259,7 @@ func TestRemoteAlreadySetGuards(t *testing.T) { adapter, ok := adapters[tt.resource] require.True(t, ok) changes := deployplan.Changes{tt.field: tt.ch} - err := addPerFieldActions(t.Context(), adapter, changes, nil, nil) + err := addPerFieldActions(t.Context(), adapter, changes, nil, nil, nil) require.NoError(t, err) assert.Equal(t, tt.expectedAction, tt.ch.Action) if tt.expectedReason != "" { @@ -271,10 +271,10 @@ func TestRemoteAlreadySetGuards(t *testing.T) { } } -// TestFollowParentRenames pins when an ID field that references a renamed parent is +// TestFollowsParentRename pins when an ID field that references a renamed parent is // re-keyed (update_id) instead of recreated: only a provided_id_field of a resource with // DoUpdateWithID, and only if it was under that parent before the rename. -func TestFollowParentRenames(t *testing.T) { +func TestFollowsParentRename(t *testing.T) { adapters, err := dresources.InitAll(nil) require.NoError(t, err) @@ -319,11 +319,11 @@ func TestFollowParentRenames(t *testing.T) { t.Run(tt.name, func(t *testing.T) { adapter, ok := adapters[tt.resource] require.True(t, ok) - ch := &deployplan.ChangeDesc{Action: deployplan.Recreate, Reason: "id_field", Old: tt.old, New: "s_new", Remote: tt.old} + ch := &deployplan.ChangeDesc{Old: tt.old, New: "s_new", Remote: tt.old} parentRenames := map[string]*deployplan.ChangeDesc{ tt.field: {Action: deployplan.UpdateWithID, Reason: "id_changes", Old: "s_old", New: "s_new", Remote: "s_old"}, } - require.NoError(t, followParentRenames(adapter, deployplan.Changes{tt.field: ch}, parentRenames)) + require.NoError(t, addPerFieldActions(t.Context(), adapter, deployplan.Changes{tt.field: ch}, nil, nil, parentRenames)) assert.Equal(t, tt.expectedAction, ch.Action) }) } @@ -380,7 +380,7 @@ func TestRecreateBackendDefault(t *testing.T) { adapter, ok := adapters[tt.resource] require.True(t, ok) changes := deployplan.Changes{tt.field: tt.ch} - require.NoError(t, addPerFieldActions(t.Context(), adapter, changes, nil, nil)) + require.NoError(t, addPerFieldActions(t.Context(), adapter, changes, nil, nil, nil)) assert.Equal(t, tt.expectedAction, tt.ch.Action) if tt.expectedReason != "" { assert.Equal(t, tt.expectedReason, tt.ch.Reason) @@ -511,7 +511,7 @@ func TestScalarZeroChange(t *testing.T) { adapter, ok := adapters["clusters"] require.True(t, ok) changes := deployplan.Changes{tt.field: tt.ch} - require.NoError(t, addPerFieldActions(t.Context(), adapter, changes, nil, nil)) + require.NoError(t, addPerFieldActions(t.Context(), adapter, changes, nil, nil, nil)) assert.Equal(t, tt.expectedAction, tt.ch.Action) if tt.expectedReason != "" { assert.Equal(t, tt.expectedReason, tt.ch.Reason) @@ -575,7 +575,7 @@ func jobRunResultStateAction(t *testing.T, state *jobs.RunState) *deployplan.Cha Remote: state.ResultState, }} - require.NoError(t, addPerFieldActions(t.Context(), adapters["job_runs"], changes, nil, remote)) + require.NoError(t, addPerFieldActions(t.Context(), adapters["job_runs"], changes, nil, remote, nil)) return changes["result_state"] } @@ -849,7 +849,7 @@ func TestLadderBackendDefaultBeforeRemoteAddition(t *testing.T) { // config-absent (Old==New==nil), present in remote: matches both rules. "enable_elastic_disk": &deployplan.ChangeDesc{Remote: true}, } - require.NoError(t, addPerFieldActions(t.Context(), adapter, changes, newState, nil)) + require.NoError(t, addPerFieldActions(t.Context(), adapter, changes, newState, nil, nil)) assert.Equal(t, deployplan.Skip, changes["enable_elastic_disk"].Action) assert.Equal(t, deployplan.ReasonBackendDefault, changes["enable_elastic_disk"].Reason) } @@ -870,7 +870,7 @@ func TestInternalIDChangesAreSkipped(t *testing.T) { } { t.Run(tt.resource+" "+name, func(t *testing.T) { changes := deployplan.Changes{tt.field: ch} - require.NoError(t, addPerFieldActions(t.Context(), adapters[tt.resource], changes, nil, nil)) + require.NoError(t, addPerFieldActions(t.Context(), adapters[tt.resource], changes, nil, nil, nil)) assert.Equal(t, deployplan.Skip, changes[tt.field].Action) }) } From 7b966d9ee19ec09a411d08a91b22340a0058c03d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 14:06:53 +0100 Subject: [PATCH 27/75] Point comments at followsParentRename Co-authored-by: Isaac --- bundle/direct/dresources/schema.go | 2 +- bundle/direct/dresources/volume.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dresources/schema.go b/bundle/direct/dresources/schema.go index 250d44fb975..0a69ff0e122 100644 --- a/bundle/direct/dresources/schema.go +++ b/bundle/direct/dresources/schema.go @@ -128,7 +128,7 @@ func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config * } // The catalog comes from config: a renamed catalog moves the schema with it, leaving id - // stale (see followParentRenames). + // stale (see followsParentRename). fullName := config.CatalogName + "." + nameFromID if entry.Changes.OnlyParentRenamed() { diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 5f9f9ccefca..6d8b4be9802 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -124,7 +124,7 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * nameFromID := items[len(items)-1] // The parents come from config: a renamed catalog or schema moves the volume with it, - // leaving id stale (see followParentRenames). + // leaving id stale (see followsParentRename). updateRequest.Name = config.CatalogName + "." + config.SchemaName + "." + nameFromID if entry.Changes.OnlyParentRenamed() { From a5b42ad975ab9339e4fddf0cabbb0e4c3c3d6201 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 14:44:40 +0100 Subject: [PATCH 28/75] direct: apply re-key-only update_id in the framework and record internal IDs there An update_id whose changes only follow a parent's rename (or a move the plan found) is marked state_only. For a resource with IDFromState, apply records it under the new ID without calling DoUpdateWithID; it is left out of the textual plan and the applied lines, and counted as unchanged. The internal ID is declared in config as internal_id_field. The framework records it from the remote state on every save, skips its changes in the plan and reads it in findMoved, replacing the InternalID method, the YAML ignore rules and the VolumeId/SchemaId assignments in the adapters. Co-authored-by: Isaac --- bundle/deployplan/action.go | 14 ++-- bundle/deployplan/plan.go | 35 ++++++--- bundle/direct/apply.go | 41 ++++++++-- bundle/direct/bundle_apply.go | 7 +- bundle/direct/bundle_plan.go | 12 ++- bundle/direct/dresources/README.md | 6 +- bundle/direct/dresources/adapter.go | 79 ++++++++++++++------ bundle/direct/dresources/config.go | 8 ++ bundle/direct/dresources/configs/schemas.yml | 12 +-- bundle/direct/dresources/configs/volumes.yml | 8 +- bundle/direct/dresources/schema.go | 25 +------ bundle/direct/dresources/volume.go | 24 +----- bundle/phases/deploy.go | 2 +- bundle/phases/destroy.go | 4 +- cmd/bundle/plan.go | 2 +- 15 files changed, 162 insertions(+), 117 deletions(-) diff --git a/bundle/deployplan/action.go b/bundle/deployplan/action.go index e53ef0d84bd..408e2a143f5 100644 --- a/bundle/deployplan/action.go +++ b/bundle/deployplan/action.go @@ -12,8 +12,8 @@ type Action struct { // Gone mirrors PlanEntry.Gone: the delete is a state-only cleanup because the // resource no longer exists remotely. Gone bool - // StateOnly mirrors PlanEntry.StateOnly: the delete is a state-only cleanup - // because the resource implements no DoDelete (deleting it has no backend effect). + // StateOnly mirrors PlanEntry.StateOnly: applying the action only changes the state + // (a delete of a resource without DoDelete, or a re-key). StateOnly bool } @@ -21,12 +21,10 @@ func (a Action) String() string { return fmt.Sprintf(" %s %s", a.ActionType.StringShort(), a.ResourceKey) } -// IsStateOnlyDelete reports whether applying this delete only drops the state entry -// without any backend call: the resource is already gone remotely (Gone) or has no -// delete operation (StateOnly). Such deletes are omitted from human output, excluded -// from the resource counts, and need no destructive-action approval. See the same -// method on PlanEntry. -func (a Action) IsStateOnlyDelete() bool { +// IsStateOnly reports whether applying this action makes no backend call and only changes the +// state (see PlanEntry.IsStateOnly). Such actions are omitted from human output and need no +// approval. +func (a Action) IsStateOnly() bool { return a.Gone || a.StateOnly } diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 99d70f7b80b..22ac5e83ae4 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -60,13 +60,18 @@ func (p *Plan) CountActions() ActionCounts { case Create: c.Create++ case Update, UpdateWithID, Resize: + if entry.StateOnly { + // A re-key changes nothing in the backend. + c.Unchanged++ + continue + } c.Change++ case Delete: // A state-only delete touches nothing in the backend and only drops the // state entry, so it is not a real action: leave it out of the tally // entirely rather than misreport it as deleted or unchanged. This matches // how `bundle destroy` counts its own deletions. - if entry.IsStateOnlyDelete() { + if entry.IsStateOnly() { continue } c.Delete++ @@ -128,23 +133,25 @@ type PlanEntry struct { // exists remotely. Applying such an entry only removes it from the state, without // calling the delete API, and approval prompts do not list it as a deletion. Gone bool `json:"gone,omitempty"` - // StateOnly is set on Delete entries for resources that implement no DoDelete: - // deleting them has no backend effect. Like Gone, applying such an entry only - // removes it from the state and it is excluded from destructive-action prompts, - // textual plan output and the deleted count — but unlike Gone it is a property of - // the resource type, not of the current remote state, so planning skips the - // remote read that Gone detection needs. + // StateOnly marks an entry whose apply has no backend effect and only changes the state: + // - a Delete of a resource that implements no DoDelete. Like Gone, applying it only + // removes it from the state and it is excluded from destructive-action prompts, + // textual plan output and the deleted count — but unlike Gone it is a property of + // the resource type, not of the current remote state, so planning skips the remote + // read that Gone detection needs. + // - an UpdateWithID that only follows a parent's rename (or a rename found by the plan): + // the backend already moved the resource, so applying it records the state under the + // ID IDFromState gives it. It is left out of the textual plan and counted as unchanged. StateOnly bool `json:"state_only,omitempty"` NewState *structvar.StructVarJSON `json:"new_state,omitempty"` RemoteState any `json:"remote_state,omitempty"` Changes Changes `json:"changes,omitempty"` } -// IsStateOnlyDelete reports whether applying this delete only drops the state entry -// without any backend call: the resource is already gone remotely (Gone) or has no -// delete operation (StateOnly). Such deletes are omitted from human output, excluded -// from the resource counts, and need no destructive-action approval. -func (e *PlanEntry) IsStateOnlyDelete() bool { +// IsStateOnly reports whether applying this entry makes no backend call and only changes the +// state: a delete of a resource already gone remotely (Gone) or without a delete operation, or a +// re-key (StateOnly). Such entries are omitted from human output and need no approval. +func (e *PlanEntry) IsStateOnly() bool { return e.Gone || e.StateOnly } @@ -186,6 +193,10 @@ const ( // it and whatever the remote reports is not drift (e.g. cluster libraries without a libraries // section, where job runs install their task libraries cluster-wide). ReasonUnmanaged = "unmanaged" + // ReasonInternalID: the resource's internal ID field (see ResourceLifecycleConfig.InternalIDField), + // which config never sets. + ReasonInternalID = "internal_id" + // ReasonParentRenamed: an ID field that references the name of a parent being renamed in // place. The backend moves the resource along with its parent, so it is re-keyed rather // than recreated. diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index f47f6c3cdb6..30f051b043a 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -55,6 +55,9 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, case deployplan.Update: return d.Update(ctx, db, oldID, newState, planEntry) case deployplan.UpdateWithID: + if planEntry.StateOnly { + return d.Rekey(ctx, db, oldID, newState, planEntry) + } return d.UpdateWithID(ctx, db, oldID, newState, planEntry) case deployplan.Resize: return d.Resize(ctx, db, oldID, newState, planEntry) @@ -93,7 +96,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn) + err = d.saveState(ctx, db, newID, newState, d.DependsOn, nil) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -191,7 +194,7 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("deleting state id=%s: %w", id, err) } } else { - err = d.saveState(ctx, db, id, newState, d.DependsOn) + err = d.saveState(ctx, db, id, newState, d.DependsOn, planEntry.RemoteState) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -240,7 +243,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn) + err = d.saveState(ctx, db, newID, newState, d.DependsOn, planEntry.RemoteState) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -261,6 +264,22 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return nil } +// Rekey records the resource under the ID its new state gives it, without calling the backend: a +// rename already moved it there (see PlanEntry.StateOnly). +func (d *DeploymentUnit) Rekey(ctx context.Context, db *dstate.DeploymentState, oldID string, newState any, planEntry *deployplan.PlanEntry) error { + newID, err := d.Adapter.IDFromState(newState) + if err != nil { + return err + } + log.Infof(ctx, "Re-keyed %s id=%#v (previously %#v)", d.ResourceKey, newID, oldID) + + err = d.saveState(ctx, db, newID, newState, d.DependsOn, planEntry.RemoteState) + if err != nil { + return fmt.Errorf("saving state id=%s: %w", newID, err) + } + return nil +} + func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, oldID string) error { oldState, err := d.loadPersistedState(db) if err != nil { @@ -333,7 +352,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = d.saveState(ctx, db, id, newState, d.DependsOn) + err = d.saveState(ctx, db, id, newState, d.DependsOn, entry.RemoteState) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -345,7 +364,19 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, // with content hashes, see dresources.CompactState) before persisting it. Fields already // smaller than a hash placeholder are persisted as is. Sensitive fields are replaced by a // placeholder value so secrets are never written to disk in plaintext. -func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { +// +// It first records the internal ID in state (see ResourceLifecycleConfig.InternalIDField), from the +// remote state the operation returned or else from planRemote, the object as the plan read it (nil +// for a new object). +func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry, planRemote any) error { + remote := d.RemoteState + if remote == nil { + remote = planRemote + } + err := d.Adapter.RecordInternalID(state, remote) + if err != nil { + return fmt.Errorf("recording internal id: %w", err) + } compacted, err := dresources.CompactState(d.Adapter.ResourceConfig(), state) if err != nil { return fmt.Errorf("compacting state: %w", err) diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index add38bf99f6..60e1b66bb47 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -115,7 +115,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } if action == deployplan.Delete { - if entry.IsStateOnlyDelete() { + if entry.IsStateOnly() { // The resource is already deleted remotely (Gone) or has no delete // operation (StateOnly); either way only remove it from the state, // without calling the delete API. @@ -130,7 +130,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // A state-only delete performs no backend operation, so don't report it, // consistent with the summary (CountActions excludes it) and the terraform // path in logDeploySummary. - if reportApplied && !entry.IsStateOnlyDelete() { + if reportApplied && !entry.IsStateOnly() { cmdio.LogString(ctx, deployplan.AppliedLine(resourceKey, action)) } return true @@ -173,7 +173,8 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // Reported before the remote-state refresh below: the resource is already // deployed at this point, so the line is accurate even if the refresh fails. - if reportApplied { + // A re-key changes nothing in the backend, so it is not reported (see PlanEntry.StateOnly). + if reportApplied && !entry.StateOnly { cmdio.LogString(ctx, deployplan.AppliedLine(resourceKey, action)) } } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 7c1b8981595..180015b6d7b 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -395,6 +395,11 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks // Found at a new ID: the state has to follow it there. action = deployplan.GetHigherAction(action, deployplan.UpdateWithID) } + if action == deployplan.UpdateWithID && adapter.HasIDFromState() && entry.Changes.OnlyParentRenamed() { + // The backend already moved the resource with its renamed parent, or the plan found it + // at its new ID: applying only records the state under the ID IDFromState gives it. + entry.StateOnly = true + } // Note, this unconditionally stores remoteState. However, it may updated post-deploy, so whether // it can be used for variable resolution depends on several factors, see canReadRemoteCache in LookupReferencePreDeploy @@ -501,7 +506,10 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change // backend-managed/input-only), or absent from RemoteType (a guaranteed-nil // placeholder, since RemapState is a dumb copy). Otherwise a coincidental // new == remote (both nil, say) wrongly skips a real local change. - if structdiff.IsEqual(ch.Remote, ch.New) && !ignoreRemoteChanges(cfg, generatedCfg, path) && !isFieldMissingInRemote(adapter, path) { + if adapter.IsInternalIDField(path) { + ch.Action = deployplan.Skip + ch.Reason = deployplan.ReasonInternalID + } else if structdiff.IsEqual(ch.Remote, ch.New) && !ignoreRemoteChanges(cfg, generatedCfg, path) && !isFieldMissingInRemote(adapter, path) { ch.Action = deployplan.Skip ch.Reason = deployplan.ReasonRemoteAlreadySet } else if allEmptyChange(ch) { @@ -744,7 +752,7 @@ func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan. // in savedState, so an unrelated object with the same name is never adopted; the create then fails // on it instead. func findMoved(ctx context.Context, adapter *dresources.Adapter, sv *structvar.StructVar, storedID string, savedState any) (string, any, error) { - if !adapter.HasInternalID() || !adapter.HasDoUpdateWithID() || hasUnresolvedIDField(adapter, sv) { + if !adapter.HasInternalID() || hasUnresolvedIDField(adapter, sv) { return "", nil, nil } savedInternalID, err := adapter.InternalID(savedState) diff --git a/bundle/direct/dresources/README.md b/bundle/direct/dresources/README.md index c31853fe6d7..373af02b13f 100644 --- a/bundle/direct/dresources/README.md +++ b/bundle/direct/dresources/README.md @@ -47,9 +47,11 @@ For resources whose create or update is asynchronous (the resource is not immedi If the API may return a slice's elements in a different order between calls (e.g., `depends_on` in job tasks, `privileges` in grants), implement `KeyedSlices` to compare elements by a natural key rather than by index. Without this, every deploy after any reordering shows phantom diffs. -## Moved resources: IDFromState / InternalID +## Renamed parents and moved resources: IDFromState / internal_id_field -If the ID is built from the resource's own fields (a UC full name) and the backend has its own ID that a rename keeps (a volume's `volume_id`), record that ID in the state type, set it from the API responses (and from `RemapState`), and implement `IDFromState` and `InternalID` together, along with `DoUpdateWithID`. When nothing is at the recorded ID, the plan reads at the ID the config gives the resource and takes what it finds only if its internal ID matches the one saved in state. This covers a rename done out of band, or by a deploy that stopped before re-keying the resource. +If the ID is built from the resource's own fields (a UC full name), implement `IDFromState`. Then an `update_id` that only follows a parent's rename is applied by the framework without calling `DoUpdateWithID` (the plan marks it `state_only`), so `DoUpdateWithID` only handles real updates. + +If the backend also has its own ID that a rename keeps (a volume's `volume_id`), add it to the state type, copy it in `RemapState`, and name it in the config as `internal_id_field`. The framework records it on every save and skips its changes in the plan. When nothing is at the recorded ID, the plan reads at the ID the config gives the resource and takes what it finds only if the internal ID matches. This covers a rename done out of band, or by a deploy that stopped before re-keying the resource. ## Empty states: IsEmptyState diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 64a32906462..1317772e8ac 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/calladapt" + "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go" @@ -120,15 +121,10 @@ type IResource interface { IsGone(remoteState any) bool // [Optional] IDFromState returns the ID the state gives the resource, for resource types whose ID - // is built from their own fields (a UC full name). Implemented together with InternalID. + // is built from their own fields (a UC full name). With it, an update_id that only follows a + // parent's rename is applied without calling DoUpdateWithID (see PlanEntry.StateOnly). // Example: func (*ResourceVolume) IDFromState(state *VolumeState) string IDFromState(state any) string - - // [Optional] InternalID returns the backend's own ID recorded in the state, which a rename keeps - // (a volume's volume_id). When the resource is not at its recorded ID, the plan looks for it at the - // ID IDFromState gives it, and takes what it finds only if the internal IDs match. - // Example: func (*ResourceVolume) InternalID(state *VolumeState) string - InternalID(state any) string } // Adapter wraps resource implementation, validates signatures and type consistency across methods @@ -153,11 +149,13 @@ type Adapter struct { doResize *calladapt.BoundCaller isGone *calladapt.BoundCaller idFromState *calladapt.BoundCaller - internalID *calladapt.BoundCaller resourceConfig *ResourceLifecycleConfig generatedResourceConfig *ResourceLifecycleConfig keyedSlices map[string]any + + // internalIDPath is InternalIDField, parsed; nil for a resource type without one. + internalIDPath *structpath.PathNode } func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceClient) (*Adapter, error) { @@ -196,7 +194,7 @@ func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceC overrideChangeDesc: nil, isGone: nil, idFromState: nil, - internalID: nil, + internalIDPath: nil, resourceConfig: GetResourceConfig(resourceType), generatedResourceConfig: GetGeneratedResourceConfig(resourceType), keyedSlices: nil, @@ -328,11 +326,6 @@ func (a *Adapter) initMethods(resource any) error { return err } - a.internalID, err = calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "InternalID") - if err != nil { - return err - } - keyedSlicesCall, err := calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "KeyedSlices") if err != nil { return err @@ -427,12 +420,22 @@ func (a *Adapter) validate() error { validations = append(validations, "IsGone remoteState", a.isGone.InTypes[0], remoteType) } - if (a.idFromState == nil) != (a.internalID == nil) { - return errors.New("IDFromState and InternalID must be implemented together") - } if a.idFromState != nil { validations = append(validations, "IDFromState state", a.idFromState.InTypes[0], stateType) - validations = append(validations, "InternalID state", a.internalID.InTypes[0], stateType) + } + + if a.resourceConfig != nil && a.resourceConfig.InternalIDField != "" { + if a.idFromState == nil || a.doUpdateWithID == nil { + return errors.New("internal_id_field requires IDFromState and DoUpdateWithID") + } + path, err := structpath.ParsePath(a.resourceConfig.InternalIDField) + if err != nil { + return fmt.Errorf("internal_id_field: %w", err) + } + if err := structaccess.ValidatePath(stateType, path); err != nil { + return fmt.Errorf("internal_id_field: %w", err) + } + a.internalIDPath = path } if a.doUpdateWithID != nil { @@ -760,9 +763,19 @@ func (a *Adapter) IsGone(remoteState any) bool { return outs[0].(bool) } -// HasInternalID reports whether the resource implements IDFromState and InternalID. +// HasIDFromState reports whether the resource implements IDFromState. +func (a *Adapter) HasIDFromState() bool { + return a.idFromState != nil +} + +// HasInternalID reports whether the resource records an internal ID (see InternalIDField). func (a *Adapter) HasInternalID() bool { - return a.internalID != nil + return a.internalIDPath != nil +} + +// IsInternalIDField reports whether path is the resource's internal ID field. +func (a *Adapter) IsInternalIDField(path *structpath.PathNode) bool { + return a.internalIDPath != nil && path.String() == a.internalIDPath.String() } // IDFromState returns the ID the state gives the resource; see IResource.IDFromState. @@ -774,13 +787,31 @@ func (a *Adapter) IDFromState(state any) (string, error) { return outs[0].(string), nil } -// InternalID returns the backend's own ID recorded in state; see IResource.InternalID. +// InternalID returns the internal ID recorded in state (see InternalIDField). func (a *Adapter) InternalID(state any) (string, error) { - outs, err := a.internalID.Call(state) - if err != nil { + value, err := structaccess.Get(state, a.internalIDPath) + if err != nil || value == nil { return "", err } - return outs[0].(string), nil + id, _ := value.(string) + return id, nil +} + +// RecordInternalID copies the internal ID from remoteState into state, which is about to be saved. +// Config never sets it, so without this the saved state would lose it. +func (a *Adapter) RecordInternalID(state, remoteState any) error { + if a.internalIDPath == nil || remoteState == nil || reflect.ValueOf(remoteState).IsNil() { + return nil + } + remapped, err := a.RemapState(remoteState) + if err != nil { + return err + } + id, err := a.InternalID(remapped) + if err != nil || id == "" { + return err + } + return structaccess.Set(state, a.internalIDPath, id) } // prepareCallRequired prepares a call and ensures the method is found. diff --git a/bundle/direct/dresources/config.go b/bundle/direct/dresources/config.go index 9b3ae9080b8..7adcc36086a 100644 --- a/bundle/direct/dresources/config.go +++ b/bundle/direct/dresources/config.go @@ -68,6 +68,13 @@ type ResourceLifecycleConfig struct { // IgnoreLocalChanges: field patterns where local changes are ignored (can't be updated via API). IgnoreLocalChanges []FieldRule `yaml:"ignore_local_changes,omitempty"` + // InternalIDField: the state field holding the backend's own ID for the resource, which a rename + // keeps (a volume's volume_id). Config never sets it: the framework records it from the remote + // state on every save and skips its changes in the plan. When the resource is missing from its + // recorded ID, the plan looks for it at the ID IDFromState gives it and takes what it finds only + // if this field matches. Requires IDFromState and DoUpdateWithID. + InternalIDField string `yaml:"internal_id_field,omitempty"` + // RecreateOnChanges: field patterns that trigger delete + create when changed. RecreateOnChanges []FieldRule `yaml:"recreate_on_changes,omitempty"` @@ -155,6 +162,7 @@ const ( var empty = ResourceLifecycleConfig{ IgnoreRemoteChanges: nil, IgnoreLocalChanges: nil, + InternalIDField: "", RecreateOnChanges: nil, ProvidedIDFields: nil, UpdatableIDFields: nil, diff --git a/bundle/direct/dresources/configs/schemas.yml b/bundle/direct/dresources/configs/schemas.yml index 6517267b988..5bb8f161038 100644 --- a/bundle/direct/dresources/configs/schemas.yml +++ b/bundle/direct/dresources/configs/schemas.yml @@ -16,12 +16,6 @@ backend_defaults: # Without this, every subsequent plan produces an Update whose payload is empty, # and UC rejects it with "UpdateSchema Nothing to update". - field: properties['unity.catalog.managed.*.defaults.*'] -ignore_local_changes: - # Recorded from the backend (see InternalID); config never sets it. - - field: schema_id - reason: internal_id -ignore_remote_changes: - # The spec does not mark schema_id output-only, so no generated rule covers a state - # recorded before schema_id was. - - field: schema_id - reason: internal_id +# Assigned by the backend and kept across renames: lets the plan find a schema renamed outside the +# bundle or by an interrupted deploy (see InternalIDField). +internal_id_field: schema_id diff --git a/bundle/direct/dresources/configs/volumes.yml b/bundle/direct/dresources/configs/volumes.yml index 3c733277d18..9e570a1f7c2 100644 --- a/bundle/direct/dresources/configs/volumes.yml +++ b/bundle/direct/dresources/configs/volumes.yml @@ -16,8 +16,6 @@ normalize_slash: # https://github.com/databricks/terraform-provider-databricks/blob/v1.65.1/catalog/resource_volume.go#L25 - field: storage_location reason: uc_strips_trailing_slash -ignore_local_changes: - # Recorded from the backend (see InternalID); config never sets it. Remote changes are - # ignored by the generated rule (the spec marks it output-only). - - field: volume_id - reason: internal_id +# Assigned by the backend and kept across renames: lets the plan find a volume renamed outside the +# bundle or by an interrupted deploy (see InternalIDField). +internal_id_field: volume_id diff --git a/bundle/direct/dresources/schema.go b/bundle/direct/dresources/schema.go index 0a69ff0e122..76a619ee8ad 100644 --- a/bundle/direct/dresources/schema.go +++ b/bundle/direct/dresources/schema.go @@ -2,7 +2,6 @@ package dresources import ( "context" - "fmt" "slices" "github.com/databricks/cli/bundle/config/resources" @@ -15,7 +14,7 @@ import ( // SchemaState is the schema's config plus its schema_id. The backend assigns schema_id and a // rename keeps it, so recording it lets the plan recognize the schema under a new name (see -// InternalID). +// internal_id_field in schemas.yml). type SchemaState struct { catalog.CreateSchema SchemaId string `json:"schema_id,omitempty"` @@ -40,7 +39,7 @@ func (*ResourceSchema) New(client *databricks.WorkspaceClient) *ResourceSchema { func (*ResourceSchema) PrepareState(input *resources.Schema) *SchemaState { return &SchemaState{ CreateSchema: input.CreateSchema, - SchemaId: "", // Recorded from the backend when the schema is created or updated. + SchemaId: "", // Recorded by the framework from the remote state (internal_id_field). } } @@ -66,7 +65,6 @@ func (r *ResourceSchema) DoCreate(ctx context.Context, config *SchemaState) (str if err != nil || response == nil { return "", nil, err } - config.SchemaId = response.SchemaId return response.FullName, response, nil } @@ -111,12 +109,11 @@ func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *Schema log.Warnf(ctx, "schemas: response contains unexpected full_name=%#v (expected %#v)", response.FullName, id) } - config.SchemaId = response.SchemaId return response, nil } // DoUpdateWithID renames the schema in place when the name changes; the ID is the full name, so it changes too. -func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config *SchemaState, entry *PlanEntry) (string, *catalog.SchemaInfo, error) { +func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config *SchemaState, _ *PlanEntry) (string, *catalog.SchemaInfo, error) { nameFromID, err := getNameFromID(id) if err != nil { return "", nil, err @@ -131,22 +128,11 @@ func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config * // stale (see followsParentRename). fullName := config.CatalogName + "." + nameFromID - if entry.Changes.OnlyParentRenamed() { - // UC already moved the schema with its catalog; only the ID changes. - remote, ok := entry.RemoteState.(*catalog.SchemaInfo) - if !ok { - return "", nil, fmt.Errorf("internal error: unexpected remote state type %T", entry.RemoteState) - } - config.SchemaId = remote.SchemaId - return fullName, nil, nil - } - response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(fullName, newName, &config.CreateSchema)) if err != nil || response == nil { return "", nil, err } - config.SchemaId = response.SchemaId return response.FullName, response, nil } @@ -155,11 +141,6 @@ func (*ResourceSchema) IDFromState(state *SchemaState) string { return state.CatalogName + "." + state.Name } -// InternalID returns the schema_id. -func (*ResourceSchema) InternalID(state *SchemaState) string { - return state.SchemaId -} - func (r *ResourceSchema) DoDelete(ctx context.Context, id string, _ *SchemaState) error { return r.client.Schemas.Delete(ctx, catalog.DeleteSchemaRequest{ FullName: id, diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 6d8b4be9802..63a978c01c2 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -16,7 +16,7 @@ import ( // VolumeState is the volume's config plus its volume_id. The backend assigns volume_id and a // rename keeps it, so recording it lets the plan recognize the volume under a new name (see -// InternalID). +// internal_id_field in volumes.yml). type VolumeState struct { catalog.CreateVolumeRequestContent VolumeId string `json:"volume_id,omitempty"` @@ -41,7 +41,7 @@ func (*ResourceVolume) New(client *databricks.WorkspaceClient) *ResourceVolume { func (*ResourceVolume) PrepareState(input *resources.Volume) *VolumeState { return &VolumeState{ CreateVolumeRequestContent: input.CreateVolumeRequestContent, - VolumeId: "", // Recorded from the backend when the volume is created or updated. + VolumeId: "", // Recorded by the framework from the remote state (internal_id_field). } } @@ -67,7 +67,6 @@ func (r *ResourceVolume) DoCreate(ctx context.Context, config *VolumeState) (str if err != nil { return "", nil, err } - config.VolumeId = response.VolumeId return response.FullName, response, nil } @@ -102,11 +101,10 @@ func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *Volume log.Warnf(ctx, "volumes: response contains unexpected full_name=%#v (expected %#v)", response.FullName, id) } - config.VolumeId = response.VolumeId return response, err } -func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config *VolumeState, entry *PlanEntry) (string, *catalog.VolumeInfo, error) { +func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config *VolumeState, _ *PlanEntry) (string, *catalog.VolumeInfo, error) { updateRequest := catalog.UpdateVolumeRequestContent{ Comment: config.Comment, Name: "", // Initialized below @@ -127,16 +125,6 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * // leaving id stale (see followsParentRename). updateRequest.Name = config.CatalogName + "." + config.SchemaName + "." + nameFromID - if entry.Changes.OnlyParentRenamed() { - // UC already moved the volume with its parent; only the ID changes. - remote, ok := entry.RemoteState.(*catalog.VolumeInfo) - if !ok { - return "", nil, fmt.Errorf("internal error: unexpected remote state type %T", entry.RemoteState) - } - config.VolumeId = remote.VolumeId - return updateRequest.Name, nil, nil - } - if config.Name != nameFromID { updateRequest.NewName = config.Name } @@ -146,7 +134,6 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * return "", nil, err } - config.VolumeId = response.VolumeId return response.FullName, response, nil } @@ -155,11 +142,6 @@ func (*ResourceVolume) IDFromState(state *VolumeState) string { return state.CatalogName + "." + state.SchemaName + "." + state.Name } -// InternalID returns the volume_id. -func (*ResourceVolume) InternalID(state *VolumeState) string { - return state.VolumeId -} - func (r *ResourceVolume) DoDelete(ctx context.Context, id string, _ *VolumeState) error { return r.client.Volumes.DeleteByName(ctx, id) } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 9c4d4b178cb..1f6831281ea 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -50,7 +50,7 @@ func approvalForDeploy(ctx context.Context, b *bundle.Bundle, plan *deployplan.P // Deletes that only clean up the state (already gone remotely, or no delete // operation) are not destructive and need no approval. - actions = slices.DeleteFunc(actions, func(a deployplan.Action) bool { return a.IsStateOnlyDelete() }) + actions = slices.DeleteFunc(actions, func(a deployplan.Action) bool { return a.IsStateOnly() }) err := checkForPreventDestroy(b, actions) if err != nil { diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 07fed65ba80..8d9fc9a3492 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -93,7 +93,7 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan. // operation) are not destructive, so they are not listed as deletions and need no // approval. In particular this makes prevent_destroy inert for state-only // resources: nothing is destroyed. - deleteActions = slices.DeleteFunc(deleteActions, func(a deployplan.Action) bool { return a.IsStateOnlyDelete() }) + deleteActions = slices.DeleteFunc(deleteActions, func(a deployplan.Action) bool { return a.IsStateOnly() }) err := checkForPreventDestroy(b, deleteActions) if err != nil { @@ -193,7 +193,7 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) { // a destruction to report. deleted := 0 for _, a := range plan.GetActions() { - if a.ActionType == deployplan.Delete && !a.IsChildResource() && !a.IsStateOnlyDelete() { + if a.ActionType == deployplan.Delete && !a.IsChildResource() && !a.IsStateOnly() { deleted++ } } diff --git a/cmd/bundle/plan.go b/cmd/bundle/plan.go index 1132834bc4d..48d3be6a783 100644 --- a/cmd/bundle/plan.go +++ b/cmd/bundle/plan.go @@ -91,7 +91,7 @@ It is useful for previewing changes before running 'bundle deploy'.`, } // A state-only delete has no backend effect; keep it in the JSON // plan but omit it from the human-readable action list. - if action.IsStateOnlyDelete() { + if action.IsStateOnly() { continue } key := strings.TrimPrefix(action.ResourceKey, "resources.") From d5ce96361423d9c5a9b2b7a518ea442b58d244e0 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 15:02:21 +0100 Subject: [PATCH 29/75] acc: renames with another field changed, and case-only renames A schema or catalog renamed with its comment changed sends both in one PATCH and both land. A case-only rename of a schema or volume is accepted by UC, which keeps the lowercase name, so the deploy succeeds and the next plan is a no-op. The test server now lowercases a renamed volume, like UC. Co-authored-by: Isaac --- .../rename_with_comment/databricks.yml.tmpl | 8 ++ .../rename_with_comment/out.test.toml | 2 + .../catalogs/rename_with_comment/output.txt | 47 +++++++ .../catalogs/rename_with_comment/script | 23 ++++ .../catalogs/rename_with_comment/test.toml | 7 + .../rename_case_only/databricks.yml.tmpl | 8 ++ .../schemas/rename_case_only/out.test.toml | 2 + .../schemas/rename_case_only/output.txt | 90 +++++++++++++ .../resources/schemas/rename_case_only/script | 25 ++++ .../schemas/rename_case_only/test.toml | 7 + .../rename_with_comment/databricks.yml.tmpl | 9 ++ .../schemas/rename_with_comment/out.test.toml | 2 + .../schemas/rename_with_comment/output.txt | 51 +++++++ .../schemas/rename_with_comment/script | 24 ++++ .../schemas/rename_with_comment/test.toml | 7 + .../rename_case_only/databricks.yml.tmpl | 13 ++ .../volumes/rename_case_only/out.test.toml | 2 + .../volumes/rename_case_only/output.txt | 126 ++++++++++++++++++ .../resources/volumes/rename_case_only/script | 27 ++++ .../volumes/rename_case_only/test.toml | 7 + 20 files changed, 487 insertions(+) create mode 100644 acceptance/bundle/resources/catalogs/rename_with_comment/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/catalogs/rename_with_comment/out.test.toml create mode 100644 acceptance/bundle/resources/catalogs/rename_with_comment/output.txt create mode 100644 acceptance/bundle/resources/catalogs/rename_with_comment/script create mode 100644 acceptance/bundle/resources/catalogs/rename_with_comment/test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_case_only/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/rename_case_only/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_case_only/output.txt create mode 100644 acceptance/bundle/resources/schemas/rename_case_only/script create mode 100644 acceptance/bundle/resources/schemas/rename_case_only/test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_with_comment/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/rename_with_comment/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_with_comment/output.txt create mode 100644 acceptance/bundle/resources/schemas/rename_with_comment/script create mode 100644 acceptance/bundle/resources/schemas/rename_with_comment/test.toml create mode 100644 acceptance/bundle/resources/volumes/rename_case_only/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/volumes/rename_case_only/out.test.toml create mode 100644 acceptance/bundle/resources/volumes/rename_case_only/output.txt create mode 100644 acceptance/bundle/resources/volumes/rename_case_only/script create mode 100644 acceptance/bundle/resources/volumes/rename_case_only/test.toml diff --git a/acceptance/bundle/resources/catalogs/rename_with_comment/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/rename_with_comment/databricks.yml.tmpl new file mode 100644 index 00000000000..90953dac6dc --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_with_comment/databricks.yml.tmpl @@ -0,0 +1,8 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + comment: c1 diff --git a/acceptance/bundle/resources/catalogs/rename_with_comment/out.test.toml b/acceptance/bundle/resources/catalogs/rename_with_comment/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_with_comment/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/rename_with_comment/output.txt b/acceptance/bundle/resources/catalogs/rename_with_comment/output.txt new file mode 100644 index 00000000000..0359d87b31a --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_with_comment/output.txt @@ -0,0 +1,47 @@ + +=== Deploy a catalog with a comment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the catalog and change its comment in one deploy: both go in one PATCH +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +>>> update_file.py databricks.yml comment: c1 comment: c2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated catalogs.catalog1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]", + "body": { + "comment": "c2", + "custom_max_retention_hours": 0, + "new_name": "test_catalog_renamed_[UNIQUE_NAME]" + } +} + +=== Both landed, so the redeploy is a no-op +>>> [CLI] catalogs get test_catalog_renamed_[UNIQUE_NAME] +{ + "name": "test_catalog_renamed_[UNIQUE_NAME]", + "comment": "c2" +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.catalog1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/catalogs/rename_with_comment/script b/acceptance/bundle/resources/catalogs/rename_with_comment/script new file mode 100644 index 00000000000..f3c87c55be4 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_with_comment/script @@ -0,0 +1,23 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a catalog with a comment" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests + +title "Rename the catalog and change its comment in one deploy: both go in one PATCH" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ +trace update_file.py databricks.yml "comment: c1" "comment: c2" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "Both landed, so the redeploy is a no-op" +trace $CLI catalogs get "test_catalog_renamed_${UNIQUE_NAME}" | jq '{name, comment}' +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/catalogs/rename_with_comment/test.toml b/acceptance/bundle/resources/catalogs/rename_with_comment/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_with_comment/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/rename_case_only/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_case_only/databricks.yml.tmpl new file mode 100644 index 00000000000..1a1d53a02c5 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_case_only/databricks.yml.tmpl @@ -0,0 +1,8 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test_schema_$UNIQUE_NAME diff --git a/acceptance/bundle/resources/schemas/rename_case_only/out.test.toml b/acceptance/bundle/resources/schemas/rename_case_only/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_case_only/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_case_only/output.txt b/acceptance/bundle/resources/schemas/rename_case_only/output.txt new file mode 100644 index 00000000000..195bbc44832 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_case_only/output.txt @@ -0,0 +1,90 @@ + +=== Deploy a schema with a lowercase name +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Change only the case of the name. UC lowercases names, so it is the same name +>>> update_file.py databricks.yml test_schema_ Test_Schema_ + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "Test_Schema_[UNIQUE_NAME]" + } + }, + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS], + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "test_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[SCHEMA_ID]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_schema_[UNIQUE_NAME]", + "new": "Test_Schema_[UNIQUE_NAME]", + "remote": "test_schema_[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]", + "remote": "[SCHEMA_ID]" + } + } + } +} + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test_schema_[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "Test_Schema_[UNIQUE_NAME]" + } +} + +>>> [CLI] schemas get main.test_schema_[UNIQUE_NAME] +{ + "full_name": "main.test_schema_[UNIQUE_NAME]", + "name": "test_schema_[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" +} + +=== Redeploy +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> errcode [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/schemas/rename_case_only/script b/acceptance/bundle/resources/schemas/rename_case_only/script new file mode 100644 index 00000000000..a15de5ccca5 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_case_only/script @@ -0,0 +1,25 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace errcode $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a lowercase name" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "main.test_schema_${UNIQUE_NAME}" | jq -r .schema_id)" SCHEMA_ID + +title "Change only the case of the name. UC lowercases names, so it is the same name" +trace update_file.py databricks.yml test_schema_ Test_Schema_ +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by effective_predictive_optimization_flag +trace errcode $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI schemas get "main.test_schema_${UNIQUE_NAME}" | jq '{full_name, name, schema_id}' + +title "Redeploy" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_case_only/test.toml b/acceptance/bundle/resources/schemas/rename_case_only/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_case_only/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/rename_with_comment/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_with_comment/databricks.yml.tmpl new file mode 100644 index 00000000000..d73aacf0d14 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_comment/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + comment: c1 diff --git a/acceptance/bundle/resources/schemas/rename_with_comment/out.test.toml b/acceptance/bundle/resources/schemas/rename_with_comment/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_comment/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_with_comment/output.txt b/acceptance/bundle/resources/schemas/rename_with_comment/output.txt new file mode 100644 index 00000000000..f685df69adc --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_comment/output.txt @@ -0,0 +1,51 @@ + +=== Deploy a schema with a comment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema and change its comment in one deploy: both go in one PATCH +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> update_file.py databricks.yml comment: c1 comment: c2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "body": { + "comment": "c2", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} + +=== Both landed, so the redeploy is a no-op +>>> [CLI] schemas get main.test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "comment": "c2", + "schema_id": "[SCHEMA_ID]" +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/schemas/rename_with_comment/script b/acceptance/bundle/resources/schemas/rename_with_comment/script new file mode 100644 index 00000000000..96cdcd68160 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_comment/script @@ -0,0 +1,24 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a comment" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "main.test-schema-${UNIQUE_NAME}" | jq -r .schema_id)" SCHEMA_ID + +title "Rename the schema and change its comment in one deploy: both go in one PATCH" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +trace update_file.py databricks.yml "comment: c1" "comment: c2" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "Both landed, so the redeploy is a no-op" +trace $CLI schemas get "main.test-schema-renamed-${UNIQUE_NAME}" | jq '{full_name, comment, schema_id}' +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_with_comment/test.toml b/acceptance/bundle/resources/schemas/rename_with_comment/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_comment/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/volumes/rename_case_only/databricks.yml.tmpl b/acceptance/bundle/resources/volumes/rename_case_only/databricks.yml.tmpl new file mode 100644 index 00000000000..ca71c34c3aa --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_case_only/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test_schema_$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/volumes/rename_case_only/out.test.toml b/acceptance/bundle/resources/volumes/rename_case_only/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_case_only/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/rename_case_only/output.txt b/acceptance/bundle/resources/volumes/rename_case_only/output.txt new file mode 100644 index 00000000000..cdbdd33c65e --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_case_only/output.txt @@ -0,0 +1,126 @@ + +=== Deploy a volume with a lowercase name +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Change only the case of the name. UC lowercases names, so it is the same name +>>> update_file.py databricks.yml name: myvolume name: MyVolume + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test_schema_[UNIQUE_NAME]", + "name": "test_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "MyVolume", + "schema_name": "test_schema_[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "main", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "main.test_schema_[UNIQUE_NAME].myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "test_schema_[UNIQUE_NAME]", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "myvolume", + "new": "MyVolume", + "remote": "myvolume" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" + } + } + } +} + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test_schema_[UNIQUE_NAME].myvolume", + "body": { + "comment": "", + "new_name": "MyVolume" + } +} + +>>> [CLI] volumes read main.test_schema_[UNIQUE_NAME].myvolume +{ + "full_name": "main.test_schema_[UNIQUE_NAME].myvolume", + "name": "myvolume", + "volume_id": "[VOLUME_ID]" +} + +=== Redeploy +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> errcode [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/volumes/rename_case_only/script b/acceptance/bundle/resources/volumes/rename_case_only/script new file mode 100644 index 00000000000..16608975431 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_case_only/script @@ -0,0 +1,27 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +VOLUME="main.test_schema_${UNIQUE_NAME}.myvolume" + +cleanup() { + trace errcode $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a volume with a lowercase name" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI volumes read "$VOLUME" | jq -r .volume_id)" VOLUME_ID + +title "Change only the case of the name. UC lowercases names, so it is the same name" +trace update_file.py databricks.yml "name: myvolume" "name: MyVolume" +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +trace errcode $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI volumes read "$VOLUME" | jq '{full_name, name, volume_id}' + +title "Redeploy" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/volumes/rename_case_only/test.toml b/acceptance/bundle/resources/volumes/rename_case_only/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_case_only/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] From b1a45e41ac34cf5bfd3abf77d6e1b0157727ec8b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 15:17:40 +0100 Subject: [PATCH 30/75] direct: match the internal ID field structurally, ahead of the skip chain Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 12 ++++++++---- bundle/direct/dresources/adapter.go | 5 +++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 180015b6d7b..de2b948f913 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -500,16 +500,20 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change return err } + // Config never sets the internal ID; the framework records it from the remote state. + if adapter.IsInternalIDField(path) { + ch.Action = deployplan.Skip + ch.Reason = deployplan.ReasonInternalID + continue + } + // RemoteAlreadySet only holds when ch.Remote is a real remote value we can compare // against ch.New. Skip it for fields whose remote value is fabricated and thus // meaningless: declared ignore_remote_changes (present in RemoteType but read back // backend-managed/input-only), or absent from RemoteType (a guaranteed-nil // placeholder, since RemapState is a dumb copy). Otherwise a coincidental // new == remote (both nil, say) wrongly skips a real local change. - if adapter.IsInternalIDField(path) { - ch.Action = deployplan.Skip - ch.Reason = deployplan.ReasonInternalID - } else if structdiff.IsEqual(ch.Remote, ch.New) && !ignoreRemoteChanges(cfg, generatedCfg, path) && !isFieldMissingInRemote(adapter, path) { + if structdiff.IsEqual(ch.Remote, ch.New) && !ignoreRemoteChanges(cfg, generatedCfg, path) && !isFieldMissingInRemote(adapter, path) { ch.Action = deployplan.Skip ch.Reason = deployplan.ReasonRemoteAlreadySet } else if allEmptyChange(ch) { diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 1317772e8ac..64c7fc22d8d 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -773,9 +773,10 @@ func (a *Adapter) HasInternalID() bool { return a.internalIDPath != nil } -// IsInternalIDField reports whether path is the resource's internal ID field. +// IsInternalIDField reports whether path is the resource's internal ID field. It matches by +// prefix, like the field rules do. func (a *Adapter) IsInternalIDField(path *structpath.PathNode) bool { - return a.internalIDPath != nil && path.String() == a.internalIDPath.String() + return a.internalIDPath != nil && path.HasPrefix(a.internalIDPath) } // IDFromState returns the ID the state gives the resource; see IResource.IDFromState. From 9ed5ded95f5aa22717bb9479723de6bf218d4667 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 15:42:02 +0100 Subject: [PATCH 31/75] acc: regenerate for state-only re-keys and recorded schema_id Co-authored-by: Isaac --- acceptance/bundle/resources/catalogs/rename/output.txt | 6 +++--- .../resources/catalogs/rename_all_out_of_band/output.txt | 2 ++ .../resources/catalogs/rename_out_of_band/output.txt | 2 ++ .../resources/schemas/rename_out_of_band/output.txt | 6 +++--- .../resources/schemas/rename_then_change/output.txt | 9 +++------ .../resources/schemas/rename_with_volume/output.txt | 4 ++-- .../resources/schemas/rename_with_volume_ref/output.txt | 6 ++---- .../volumes/rename_catalog_and_schema/output.txt | 4 ++-- .../resources/volumes/rename_out_of_band/output.txt | 4 ++-- 9 files changed, 21 insertions(+), 22 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index d5910306690..e07288217d4 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -74,6 +74,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } ], "action": "update_id", + "state_only": true, "new_state": { "value": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", @@ -118,6 +119,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } ], "action": "update_id", + "state_only": true, "new_state": { "value": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", @@ -159,10 +161,8 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated catalogs.catalog1 -Updated schemas.schema1 -Updated volumes.volume1 Files: 1 uploaded, 0 deleted -Resources: 0 created, 3 changed, 0 deleted, 0 unchanged +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged >>> print_requests.py //unity-catalog { diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt index c57f61199d1..bd744dd4686 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt @@ -59,6 +59,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } ], "action": "update_id", + "state_only": true, "new_state": { "value": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", @@ -111,6 +112,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } ], "action": "update_id", + "state_only": true, "new_state": { "value": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt index fccb0c8b17b..350b55d2a5e 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt @@ -45,6 +45,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } ], "action": "update_id", + "state_only": true, "new_state": { "value": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", @@ -90,6 +91,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } ], "action": "update_id", + "state_only": true, "new_state": { "value": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt b/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt index afc72aaab04..d648ff373ce 100644 --- a/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt @@ -29,6 +29,7 @@ Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged "resources.schemas.schema1": { "id": "main.test-schema-renamed-[UNIQUE_NAME]", "action": "update_id", + "state_only": true, "new_state": { "value": { "catalog_name": "main", @@ -72,6 +73,7 @@ Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged } ], "action": "update_id", + "state_only": true, "new_state": { "value": { "catalog_name": "main", @@ -112,10 +114,8 @@ Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged === Deploy only records the new names in the state >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... -Updated schemas.schema1 -Updated volumes.volume1 Files: 3 uploaded, 0 deleted -Resources: 0 created, 2 changed, 0 deleted, 0 unchanged +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/schemas/rename_then_change/output.txt b/acceptance/bundle/resources/schemas/rename_then_change/output.txt index ed430978177..a3f6f27efc7 100644 --- a/acceptance/bundle/resources/schemas/rename_then_change/output.txt +++ b/acceptance/bundle/resources/schemas/rename_then_change/output.txt @@ -43,9 +43,8 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated schemas.schema1 -Updated volumes.volume1 Files: 1 uploaded, 0 deleted -Resources: 0 created, 2 changed, 0 deleted, 1 unchanged +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged >>> print_requests.py //unity-catalog { @@ -131,9 +130,8 @@ Resources: 1 created, 0 changed, 1 deleted, 2 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated schemas.schema2 -Updated volumes.volume1 Files: 1 uploaded, 0 deleted -Resources: 0 created, 2 changed, 0 deleted, 1 unchanged +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged >>> print_requests.py //unity-catalog { @@ -219,9 +217,8 @@ Resources: 1 created, 0 changed, 1 deleted, 2 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated schemas.schema1 -Updated volumes.volume1 Files: 1 uploaded, 0 deleted -Resources: 0 created, 2 changed, 0 deleted, 1 unchanged +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged >>> print_requests.py //unity-catalog { diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt index c49df8092b0..bb058ef40fd 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -77,6 +77,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } ], "action": "update_id", + "state_only": true, "new_state": { "value": { "catalog_name": "main", @@ -118,9 +119,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated schemas.schema1 -Updated volumes.volume1 Files: 1 uploaded, 0 deleted -Resources: 0 created, 2 changed, 0 deleted, 0 unchanged +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> print_requests.py //unity-catalog { diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt index ceb77a34e65..8807308a705 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt @@ -31,18 +31,16 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan update jobs.job1 update schemas.schema1 -update volumes.volume1 -Plan: 0 to add, 3 to change, 0 to delete, 0 unchanged +Plan: 0 to add, 2 to change, 0 to delete, 1 unchanged === Deploy passes the volume's new full_name and id to the job >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated jobs.job1 Updated schemas.schema1 -Updated volumes.volume1 Files: 2 uploaded, 0 deleted -Resources: 0 created, 3 changed, 0 deleted, 0 unchanged +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged >>> print_requests.py //unity-catalog { diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt index a0fac10b722..e8fd1ecfdde 100644 --- a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt @@ -100,6 +100,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } ], "action": "update_id", + "state_only": true, "new_state": { "value": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", @@ -149,9 +150,8 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated catalogs.catalog1 Updated schemas.schema1 -Updated volumes.volume1 Files: 2 uploaded, 0 deleted -Resources: 0 created, 3 changed, 0 deleted, 0 unchanged +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged >>> print_requests.py //unity-catalog { diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt b/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt index d9309f73e35..f62a0c3920a 100644 --- a/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt @@ -57,6 +57,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged } ], "action": "update_id", + "state_only": true, "new_state": { "value": { "catalog_name": "main", @@ -97,9 +98,8 @@ Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged === Deploy only records the new name in the state >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... -Updated volumes.volume1 Files: 3 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 1 unchanged +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //unity-catalog From 3f8a5ad95525bcd9c61a845a284786238391ad40 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 16:42:15 +0100 Subject: [PATCH 32/75] deployplan: name the reason at the call site with Changes.HasOnlySingleReason Co-authored-by: Isaac --- bundle/deployplan/plan.go | 8 ++++---- bundle/deployplan/plan_test.go | 11 +++++++++-- bundle/direct/bundle_plan.go | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 22ac5e83ae4..af154ab8740 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -248,11 +248,11 @@ func (c *Changes) HasChangeExcept(prefixes ...string) bool { return false } -// OnlyParentRenamed reports whether every actionable change follows a parent's rename -// (ReasonParentRenamed). The backend has already moved such a resource, so only its ID changes. -func (c *Changes) OnlyParentRenamed() bool { +// HasOnlySingleReason reports whether every actionable change has the given reason. It also holds +// when no change is actionable. +func (c *Changes) HasOnlySingleReason(reason string) bool { for _, change := range *c { - if change.Action != Skip && change.Reason != ReasonParentRenamed { + if change.Action != Skip && change.Reason != reason { return false } } diff --git a/bundle/deployplan/plan_test.go b/bundle/deployplan/plan_test.go index dad1020fcfb..1d93583514a 100644 --- a/bundle/deployplan/plan_test.go +++ b/bundle/deployplan/plan_test.go @@ -66,7 +66,7 @@ func TestHasChange(t *testing.T) { } } -func TestOnlyParentRenamed(t *testing.T) { +func TestHasOnlySingleReason(t *testing.T) { parentRenamed := &deployplan.ChangeDesc{Action: deployplan.UpdateWithID, Reason: deployplan.ReasonParentRenamed} tests := []struct { name string @@ -99,10 +99,17 @@ func TestOnlyParentRenamed(t *testing.T) { changes: deployplan.Changes{"name": {Action: deployplan.UpdateWithID, Reason: "id_changes"}}, want: false, }, + { + name: "only skipped changes", + changes: deployplan.Changes{ + "name": {Action: deployplan.Skip, Reason: deployplan.ReasonRemoteAlreadySet}, + }, + want: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, tt.changes.OnlyParentRenamed()) + assert.Equal(t, tt.want, tt.changes.HasOnlySingleReason(deployplan.ReasonParentRenamed)) }) } } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index de2b948f913..ce36c7e3c56 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -395,7 +395,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks // Found at a new ID: the state has to follow it there. action = deployplan.GetHigherAction(action, deployplan.UpdateWithID) } - if action == deployplan.UpdateWithID && adapter.HasIDFromState() && entry.Changes.OnlyParentRenamed() { + if action == deployplan.UpdateWithID && adapter.HasIDFromState() && entry.Changes.HasOnlySingleReason(deployplan.ReasonParentRenamed) { // The backend already moved the resource with its renamed parent, or the plan found it // at its new ID: applying only records the state under the ID IDFromState gives it. entry.StateOnly = true From f37978e21c7133b23927672447c71650fc781df9 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 21:32:45 +0100 Subject: [PATCH 33/75] acc: mark the catalog out-of-band rename tests with Badness (pre-existing on main) Co-authored-by: Isaac --- .../bundle/resources/catalogs/rename_all_out_of_band/test.toml | 1 + .../bundle/resources/catalogs/rename_out_of_band/test.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml index 4272a80cf61..698c1c6ca49 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml @@ -1,3 +1,4 @@ +Badness = "The plan does not look for a catalog renamed outside the bundle, so it creates it again: the create fails with CATALOG_ALREADY_EXISTS on every deploy, and destroy leaves the renamed catalog behind. Same on main." Cloud = true Ignore = [ diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml index 4272a80cf61..698c1c6ca49 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml @@ -1,3 +1,4 @@ +Badness = "The plan does not look for a catalog renamed outside the bundle, so it creates it again: the create fails with CATALOG_ALREADY_EXISTS on every deploy, and destroy leaves the renamed catalog behind. Same on main." Cloud = true Ignore = [ From b09c34653ee577df5d539a8d579b0b3b870911e1 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 11:31:31 +0100 Subject: [PATCH 34/75] acc: drop errcode from commands that do not fail Co-authored-by: Isaac --- .../resources/catalogs/rename_all_out_of_band/output.txt | 4 ++-- .../bundle/resources/catalogs/rename_all_out_of_band/script | 4 ++-- .../bundle/resources/catalogs/rename_out_of_band/output.txt | 4 ++-- .../bundle/resources/catalogs/rename_out_of_band/script | 4 ++-- .../bundle/resources/schemas/rename_case_only/output.txt | 4 ++-- acceptance/bundle/resources/schemas/rename_case_only/script | 4 ++-- .../bundle/resources/volumes/rename_case_only/output.txt | 4 ++-- acceptance/bundle/resources/volumes/rename_case_only/script | 4 ++-- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt index bd744dd4686..feb14bcca39 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt @@ -180,9 +180,9 @@ Error: cannot update_id resources.volumes.volume1: dependency failed: resources. Files: 3 uploaded, 0 deleted ->>> errcode [CLI] bundle destroy --auto-approve +>>> [CLI] bundle destroy --auto-approve All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default Destroy: 0 deleted ->>> errcode [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force +>>> [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script index 9645fc0dc02..aa51aadefb6 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script @@ -4,9 +4,9 @@ OLD_CATALOG="test_catalog_${UNIQUE_NAME}" NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" cleanup() { - trace errcode $CLI bundle destroy --auto-approve + trace $CLI bundle destroy --auto-approve # The renamed catalog is not where the state looks for it, so destroy cannot remove it. - trace errcode $CLI catalogs delete "$NEW_CATALOG" --force + trace $CLI catalogs delete "$NEW_CATALOG" --force # Consume the destroy's requests so none are left recorded. print_requests.py //unity-catalog &> LOG.destroy_requests diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt index 350b55d2a5e..529021aa62d 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt @@ -145,9 +145,9 @@ Error: cannot update_id resources.volumes.volume1: dependency failed: resources. Files: 3 uploaded, 0 deleted ->>> errcode [CLI] bundle destroy --auto-approve +>>> [CLI] bundle destroy --auto-approve All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default Destroy: 0 deleted ->>> errcode [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force +>>> [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_out_of_band/script index 8b161d76a37..1c39637d66b 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/script @@ -4,9 +4,9 @@ OLD_CATALOG="test_catalog_${UNIQUE_NAME}" NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" cleanup() { - trace errcode $CLI bundle destroy --auto-approve + trace $CLI bundle destroy --auto-approve # The renamed catalog is not where the state looks for it, so destroy cannot remove it. - trace errcode $CLI catalogs delete "$NEW_CATALOG" --force + trace $CLI catalogs delete "$NEW_CATALOG" --force # Consume the destroy's requests so none are left recorded. print_requests.py //unity-catalog &> LOG.destroy_requests diff --git a/acceptance/bundle/resources/schemas/rename_case_only/output.txt b/acceptance/bundle/resources/schemas/rename_case_only/output.txt index 195bbc44832..cdeaaeab649 100644 --- a/acceptance/bundle/resources/schemas/rename_case_only/output.txt +++ b/acceptance/bundle/resources/schemas/rename_case_only/output.txt @@ -50,7 +50,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } ->>> errcode [CLI] bundle deploy +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated schemas.schema1 Files: 2 uploaded, 0 deleted @@ -78,7 +78,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> errcode [CLI] bundle destroy --auto-approve +>>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.schemas.schema1 diff --git a/acceptance/bundle/resources/schemas/rename_case_only/script b/acceptance/bundle/resources/schemas/rename_case_only/script index a15de5ccca5..9e34bb1b159 100644 --- a/acceptance/bundle/resources/schemas/rename_case_only/script +++ b/acceptance/bundle/resources/schemas/rename_case_only/script @@ -1,7 +1,7 @@ envsubst < databricks.yml.tmpl > databricks.yml cleanup() { - trace errcode $CLI bundle destroy --auto-approve + trace $CLI bundle destroy --auto-approve # Consume the destroy's requests so none are left recorded. print_requests.py //unity-catalog &> LOG.destroy_requests @@ -17,7 +17,7 @@ title "Change only the case of the name. UC lowercases names, so it is the same trace update_file.py databricks.yml test_schema_ Test_Schema_ # Drop the remote_state fields that differ between clouds or from the test server. trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by effective_predictive_optimization_flag -trace errcode $CLI bundle deploy +trace $CLI bundle deploy trace print_requests.py //unity-catalog trace $CLI schemas get "main.test_schema_${UNIQUE_NAME}" | jq '{full_name, name, schema_id}' diff --git a/acceptance/bundle/resources/volumes/rename_case_only/output.txt b/acceptance/bundle/resources/volumes/rename_case_only/output.txt index cdbdd33c65e..d0d42aa4476 100644 --- a/acceptance/bundle/resources/volumes/rename_case_only/output.txt +++ b/acceptance/bundle/resources/volumes/rename_case_only/output.txt @@ -80,7 +80,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } ->>> errcode [CLI] bundle deploy +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated volumes.volume1 Files: 2 uploaded, 0 deleted @@ -107,7 +107,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged ->>> errcode [CLI] bundle destroy --auto-approve +>>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.schemas.schema1 delete resources.volumes.volume1 diff --git a/acceptance/bundle/resources/volumes/rename_case_only/script b/acceptance/bundle/resources/volumes/rename_case_only/script index 16608975431..ebaf4e89b14 100644 --- a/acceptance/bundle/resources/volumes/rename_case_only/script +++ b/acceptance/bundle/resources/volumes/rename_case_only/script @@ -3,7 +3,7 @@ envsubst < databricks.yml.tmpl > databricks.yml VOLUME="main.test_schema_${UNIQUE_NAME}.myvolume" cleanup() { - trace errcode $CLI bundle destroy --auto-approve + trace $CLI bundle destroy --auto-approve # Consume the destroy's requests so none are left recorded. print_requests.py //unity-catalog &> LOG.destroy_requests @@ -19,7 +19,7 @@ title "Change only the case of the name. UC lowercases names, so it is the same trace update_file.py databricks.yml "name: myvolume" "name: MyVolume" # Drop the remote_state fields that differ between clouds or from the test server. trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location -trace errcode $CLI bundle deploy +trace $CLI bundle deploy trace print_requests.py //unity-catalog trace $CLI volumes read "$VOLUME" | jq '{full_name, name, volume_id}' From 40d7b23a8d4050f44d6b97064bdf36d1f0052fab Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 14:21:56 +0100 Subject: [PATCH 35/75] Shorten the schema rename changelog entry Co-authored-by: Isaac --- .nextchanges/bundles/schema-rename.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/schema-rename.md b/.nextchanges/bundles/schema-rename.md index 2ac01106dbe..3bc837ae5d1 100644 --- a/.nextchanges/bundles/schema-rename.md +++ b/.nextchanges/bundles/schema-rename.md @@ -1 +1 @@ -* Rename a schema in place when its `name` changes, instead of deleting and recreating it. Schemas and volumes inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. ([#7343](https://github.com/databricks/cli/pull/7343)) +* Rename a schema in place when its `name` changes, instead of deleting and recreating it. Schemas and volumes inside a renamed catalog or schema now follow it to the new name. ([#7343](https://github.com/databricks/cli/pull/7343)) From a3c1f0b7cc3c859fe1737d0e925b6cce4fa091fd Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 15:02:41 +0100 Subject: [PATCH 36/75] acc: show the whole plan in the interrupted tests, without timestamps; record volume_id in empty-resource-id The volume's created_at/updated_at made rename_interrupted flaky ([UNIX_TIME_MILLIS] vs [UNIX_TIME_MILLIS][0] depending on timing). Co-authored-by: Isaac --- .../schemas/recreate_interrupted/output.txt | 76 +++++++--- .../schemas/recreate_interrupted/script | 2 +- .../schemas/rename_interrupted/output.txt | 139 ++++++++++++------ .../schemas/rename_interrupted/script | 2 +- 4 files changed, 146 insertions(+), 73 deletions(-) diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt b/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt index 8d92949d3e0..3be652c4773 100644 --- a/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt @@ -48,32 +48,64 @@ Exit code: [KILLED] === The volume went with the old schema. Nothing with its volume_id is at its new path, so the plan creates it >>> [CLI] bundle plan --output json { - "depends_on": [ - { - "node": "resources.schemas.schema1", - "label": "${resources.schemas.schema1.name}" - } - ], - "action": "create", - "new_state": { - "value": { + "resources.schemas.schema1": { + "action": "skip", + "remote_state": { + "browse_only": false, "catalog_name": "main", - "name": "myvolume", - "schema_name": "test-schema-renamed-[UNIQUE_NAME]", - "volume_type": "MANAGED" + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "test-schema-renamed-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "storage_root": "s3://mybucket/b", + "updated_by": "[USERNAME]" + }, + "changes": { + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } } }, - "changes": { - "schema_name": { - "action": "recreate", - "reason": "id_field", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]" + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } }, - "volume_id": { - "action": "skip", - "reason": "internal_id", - "old": "[VOLUME_ID]" + "changes": { + "schema_name": { + "action": "recreate", + "reason": "id_field", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]" + } } } } diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/script b/acceptance/bundle/resources/schemas/recreate_interrupted/script index df501ef6124..2d76a50db54 100644 --- a/acceptance/bundle/resources/schemas/recreate_interrupted/script +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/script @@ -25,7 +25,7 @@ trace errcode $CLI bundle deploy -q --auto-approve trace print_requests.py //unity-catalog title "The volume went with the old schema. Nothing with its volume_id is at its new path, so the plan creates it" -trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"]' +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at title "The next deploy creates a new volume" trace $CLI bundle deploy --force-lock diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt index 6c768de425a..74f616d68b7 100644 --- a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt +++ b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt @@ -38,61 +38,102 @@ Exit code: [KILLED] >>> [CLI] bundle plan --output json { - "id": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", - "depends_on": [ - { - "node": "resources.schemas.schema1", - "label": "${resources.schemas.schema1.name}" + "resources.schemas.schema1": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "comment": "", + "created_by": "[USERNAME]", + "custom_max_retention_hours": 0, + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "test-schema-renamed-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_by": "[USERNAME]" + }, + "changes": { + "comment": { + "action": "skip", + "reason": "empty", + "remote": "" + }, + "custom_max_retention_hours": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } } - ], - "action": "update_id", - "new_state": { - "value": { + }, + "resources.volumes.volume1": { + "id": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "comment": "c2", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "remote_state": { "catalog_name": "main", - "comment": "c2", + "comment": "c1", + "created_by": "[USERNAME]", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", "name": "myvolume", + "owner": "[USERNAME]", "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[VOLUME_ID]", + "volume_id": "[VOLUME_ID]", "volume_type": "MANAGED" - } - }, - "remote_state": { - "catalog_name": "main", - "comment": "c1", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", - "name": "myvolume", - "owner": "[USERNAME]", - "schema_name": "test-schema-renamed-[UNIQUE_NAME]", - "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[VOLUME_ID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "volume_id": "[VOLUME_ID]", - "volume_type": "MANAGED" - }, - "changes": { - "comment": { - "action": "update", - "old": "c1", - "new": "c2", - "remote": "c1" - }, - "schema_name": { - "action": "skip", - "reason": "remote_already_set", - "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]", - "remote": "test-schema-renamed-[UNIQUE_NAME]" - }, - "storage_location": { - "action": "skip", - "reason": "immutable_backend_value", - "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[VOLUME_ID]" }, - "volume_id": { - "action": "skip", - "reason": "internal_id", - "old": "[VOLUME_ID]", - "remote": "[VOLUME_ID]" + "changes": { + "comment": { + "action": "update", + "old": "c1", + "new": "c2", + "remote": "c1" + }, + "schema_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-renamed-[UNIQUE_NAME]" + }, + "storage_location": { + "action": "skip", + "reason": "immutable_backend_value", + "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[VOLUME_ID]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" + } } } } diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/script b/acceptance/bundle/resources/schemas/rename_interrupted/script index 76ee184840a..c829d0dea23 100644 --- a/acceptance/bundle/resources/schemas/rename_interrupted/script +++ b/acceptance/bundle/resources/schemas/rename_interrupted/script @@ -25,7 +25,7 @@ trace print_requests.py //unity-catalog title "The volume is gone from its recorded path, so the plan looks for it at its path in config" trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, volume_id: .state.volume_id}' -trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"]' +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at title "The next deploy updates the volume at its new path" trace $CLI bundle deploy --force-lock From 1e6a0153d0cc6c52c525ed5794f3075f8b14b041 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 14:30:09 +0100 Subject: [PATCH 37/75] Apply suggestion from @janniklasrose Co-authored-by: Jan N Rose --- cmd/bundle/plan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/bundle/plan.go b/cmd/bundle/plan.go index 48d3be6a783..b61208eb6de 100644 --- a/cmd/bundle/plan.go +++ b/cmd/bundle/plan.go @@ -89,7 +89,7 @@ It is useful for previewing changes before running 'bundle deploy'.`, if action.ActionType == deployplan.Skip { continue } - // A state-only delete has no backend effect; keep it in the JSON + // A state-only action has no backend effect; keep it in the JSON // plan but omit it from the human-readable action list. if action.IsStateOnly() { continue From f952d4f5beefee99abce58e2fdeb56c775ba5013 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 15:08:20 +0100 Subject: [PATCH 38/75] acc: a volume recreated while its schema is renamed (Badness) The recreate deletes the volume at its recorded path after the schema's rename moved it, so the moved volume is left behind. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 16 ++ .../move_from_renamed_schema/out.test.toml | 2 + .../move_from_renamed_schema/output.txt | 192 ++++++++++++++++++ .../volumes/move_from_renamed_schema/script | 34 ++++ .../move_from_renamed_schema/test.toml | 8 + .../databricks.yml.tmpl | 14 ++ .../recreate_in_renamed_schema/out.test.toml | 2 + .../recreate_in_renamed_schema/output.txt | 174 ++++++++++++++++ .../volumes/recreate_in_renamed_schema/script | 23 +++ .../recreate_in_renamed_schema/test.toml | 10 + 10 files changed, 475 insertions(+) create mode 100644 acceptance/bundle/resources/volumes/move_from_renamed_schema/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/volumes/move_from_renamed_schema/out.test.toml create mode 100644 acceptance/bundle/resources/volumes/move_from_renamed_schema/output.txt create mode 100644 acceptance/bundle/resources/volumes/move_from_renamed_schema/script create mode 100644 acceptance/bundle/resources/volumes/move_from_renamed_schema/test.toml create mode 100644 acceptance/bundle/resources/volumes/recreate_in_renamed_schema/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/volumes/recreate_in_renamed_schema/out.test.toml create mode 100644 acceptance/bundle/resources/volumes/recreate_in_renamed_schema/output.txt create mode 100644 acceptance/bundle/resources/volumes/recreate_in_renamed_schema/script create mode 100644 acceptance/bundle/resources/volumes/recreate_in_renamed_schema/test.toml diff --git a/acceptance/bundle/resources/volumes/move_from_renamed_schema/databricks.yml.tmpl b/acceptance/bundle/resources/volumes/move_from_renamed_schema/databricks.yml.tmpl new file mode 100644 index 00000000000..01d93162314 --- /dev/null +++ b/acceptance/bundle/resources/volumes/move_from_renamed_schema/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema1-$UNIQUE_NAME + schema2: + catalog_name: main + name: test-schema2-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/volumes/move_from_renamed_schema/out.test.toml b/acceptance/bundle/resources/volumes/move_from_renamed_schema/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/volumes/move_from_renamed_schema/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/move_from_renamed_schema/output.txt b/acceptance/bundle/resources/volumes/move_from_renamed_schema/output.txt new file mode 100644 index 00000000000..e89a5e4a7fb --- /dev/null +++ b/acceptance/bundle/resources/volumes/move_from_renamed_schema/output.txt @@ -0,0 +1,192 @@ + +=== Deploy two schemas and a volume in the first one +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created schemas.schema2 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the first schema and move the volume to the second one in one deploy. Moving the volume recreates it +>>> update_file.py databricks.yml test-schema1- test-schema1-renamed- + +>>> update_file.py databricks.yml schemas.schema1.name schemas.schema2.name + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "test-schema1-renamed-[UNIQUE_NAME]" + } + }, + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema1-[UNIQUE_NAME]", + "name": "test-schema1-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema1-[UNIQUE_NAME]", + "new": "test-schema1-renamed-[UNIQUE_NAME]", + "remote": "test-schema1-[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } + } + }, + "resources.schemas.schema2": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema2-[UNIQUE_NAME]", + "name": "test-schema2-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema2", + "label": "${resources.schemas.schema2.name}" + } + ], + "action": "recreate", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema2-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "main", + "created_by": "[USERNAME]", + "full_name": "main.test-schema1-[UNIQUE_NAME].myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "test-schema1-[UNIQUE_NAME]", + "volume_id": "[OLD_VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "schema_name": { + "action": "recreate", + "reason": "id_field", + "old": "test-schema1-[UNIQUE_NAME]", + "new": "test-schema2-[UNIQUE_NAME]", + "remote": "test-schema1-[UNIQUE_NAME]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[OLD_VOLUME_ID]", + "remote": "[OLD_VOLUME_ID]" + } + } + } +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + recreate resources.volumes.volume1 +Recreated volumes.volume1 +Updated schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 1 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema1-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema1-renamed-[UNIQUE_NAME]" + } +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema1-[UNIQUE_NAME].myvolume" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema2-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +=== The volume is in the second schema +>>> [CLI] volumes read main.test-schema2-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema2-[UNIQUE_NAME].myvolume" +} + +=== The old volume moved with the renamed schema. It should have been deleted +>>> [CLI] volumes read main.test-schema1-renamed-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema1-renamed-[UNIQUE_NAME].myvolume", + "volume_id": "[OLD_VOLUME_ID]" +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.schemas.schema2 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + delete resources.schemas.schema2 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/volumes/move_from_renamed_schema/script b/acceptance/bundle/resources/volumes/move_from_renamed_schema/script new file mode 100644 index 00000000000..1f4b6782023 --- /dev/null +++ b/acceptance/bundle/resources/volumes/move_from_renamed_schema/script @@ -0,0 +1,34 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +RENAMED_VOLUME="main.test-schema1-renamed-${UNIQUE_NAME}.myvolume" +NEW_VOLUME="main.test-schema2-${UNIQUE_NAME}.myvolume" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy two schemas and a volume in the first one" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI volumes read "main.test-schema1-${UNIQUE_NAME}.myvolume" | jq -r .volume_id)" OLD_VOLUME_ID + +title "Rename the first schema and move the volume to the second one in one deploy. Moving the volume recreates it" +trace update_file.py databricks.yml test-schema1- test-schema1-renamed- +trace update_file.py databricks.yml schemas.schema1.name schemas.schema2.name +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +trace $CLI bundle deploy --auto-approve +trace print_requests.py //unity-catalog + +title "The volume is in the second schema" +trace $CLI volumes read "$NEW_VOLUME" | jq '{full_name}' + +title "The old volume moved with the renamed schema. It should have been deleted" +trace $CLI volumes read "$RENAMED_VOLUME" | jq '{full_name, volume_id}' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/volumes/move_from_renamed_schema/test.toml b/acceptance/bundle/resources/volumes/move_from_renamed_schema/test.toml new file mode 100644 index 00000000000..4bf1280df97 --- /dev/null +++ b/acceptance/bundle/resources/volumes/move_from_renamed_schema/test.toml @@ -0,0 +1,8 @@ +Badness = "The schema's rename moves the volume, but the recreate deletes the volume at its recorded path, which no longer exists. The volume moved with the schema is left behind untracked." +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/databricks.yml.tmpl b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/databricks.yml.tmpl new file mode 100644 index 00000000000..abd22ee308d --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume + volume_type: MANAGED diff --git a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/out.test.toml b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/output.txt b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/output.txt new file mode 100644 index 00000000000..1c07fe27277 --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/output.txt @@ -0,0 +1,174 @@ + +=== Deploy a schema with a managed volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema and change the volume's type in one deploy. The type change recreates the volume +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> update_file.py databricks.yml volume_type: MANAGED volume_type: EXTERNAL + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "test-schema-renamed-[UNIQUE_NAME]" + } + }, + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "test-schema-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_by": "[USERNAME]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "recreate", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "EXTERNAL" + } + }, + "remote_state": { + "catalog_name": "main", + "created_by": "[USERNAME]", + "full_name": "main.test-schema-[UNIQUE_NAME].myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "test-schema-[UNIQUE_NAME]", + "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", + "volume_id": "[UUID]", + "volume_type": "MANAGED" + }, + "changes": { + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + }, + "storage_location": { + "action": "skip", + "reason": "immutable_backend_value", + "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + }, + "volume_type": { + "action": "recreate", + "reason": "immutable", + "old": "MANAGED", + "new": "EXTERNAL", + "remote": "MANAGED" + } + } + } +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + recreate resources.volumes.volume1 +Recreated volumes.volume1 +Updated schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 1 changed, 1 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema-[UNIQUE_NAME].myvolume" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "EXTERNAL" + } +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/script b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/script new file mode 100644 index 00000000000..57d49e071b5 --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/script @@ -0,0 +1,23 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a managed volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests + +title "Rename the schema and change the volume's type in one deploy. The type change recreates the volume" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +trace update_file.py databricks.yml "volume_type: MANAGED" "volume_type: EXTERNAL" +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at +trace $CLI bundle deploy --auto-approve +trace print_requests.py //unity-catalog + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/test.toml b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/test.toml new file mode 100644 index 00000000000..e9adadf175d --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/test.toml @@ -0,0 +1,10 @@ +Badness = "The schema's rename moves the volume, but the recreate deletes the volume at its recorded path, which no longer exists, and then creates it under the name the moved volume still has. The test server accepts that create, overwriting the moved volume; a real workspace would reject it because the name is taken." + +# Local only: an EXTERNAL volume needs an external location, which the test workspaces do not have. +# volumes/move_from_renamed_schema covers the same recreate on a real workspace. + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] From 55689e677aeb07f9d74e3cd668e77d8c4cb6385c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 16:17:28 +0100 Subject: [PATCH 39/75] direct: recreate a resource where its renamed parent moved it A parent renamed in the same plan moves the resource before its recreate runs. The recreate now deletes it at the ID the rename gives it, after the rename. The parent comes from the state's depends_on, since the config may no longer reference it. Co-authored-by: Isaac --- .../move_from_renamed_schema/output.txt | 16 ++-- .../volumes/move_from_renamed_schema/script | 4 +- .../move_from_renamed_schema/test.toml | 1 - .../recreate_in_renamed_schema/output.txt | 3 +- .../recreate_in_renamed_schema/test.toml | 1 - bundle/deployplan/plan.go | 5 +- bundle/direct/bundle_plan.go | 89 +++++++++++++++++++ bundle/direct/bundle_plan_test.go | 77 ++++++++++++++++ 8 files changed, 182 insertions(+), 14 deletions(-) diff --git a/acceptance/bundle/resources/volumes/move_from_renamed_schema/output.txt b/acceptance/bundle/resources/volumes/move_from_renamed_schema/output.txt index e89a5e4a7fb..001063266cf 100644 --- a/acceptance/bundle/resources/volumes/move_from_renamed_schema/output.txt +++ b/acceptance/bundle/resources/volumes/move_from_renamed_schema/output.txt @@ -73,10 +73,15 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } }, "resources.volumes.volume1": { + "id": "main.test-schema1-renamed-[UNIQUE_NAME].myvolume", "depends_on": [ { "node": "resources.schemas.schema2", "label": "${resources.schemas.schema2.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" } ], "action": "recreate", @@ -141,7 +146,7 @@ Resources: 1 created, 1 changed, 1 deleted, 1 unchanged } { "method": "DELETE", - "path": "/api/2.1/unity-catalog/volumes/main.test-schema1-[UNIQUE_NAME].myvolume" + "path": "/api/2.1/unity-catalog/volumes/main.test-schema1-renamed-[UNIQUE_NAME].myvolume" } { "method": "POST", @@ -160,12 +165,9 @@ Resources: 1 created, 1 changed, 1 deleted, 1 unchanged "full_name": "main.test-schema2-[UNIQUE_NAME].myvolume" } -=== The old volume moved with the renamed schema. It should have been deleted ->>> [CLI] volumes read main.test-schema1-renamed-[UNIQUE_NAME].myvolume -{ - "full_name": "main.test-schema1-renamed-[UNIQUE_NAME].myvolume", - "volume_id": "[OLD_VOLUME_ID]" -} +=== The old volume, moved with the renamed schema, was deleted there +>>> musterr [CLI] volumes read main.test-schema1-renamed-[UNIQUE_NAME].myvolume +Error: Volume 'main.test-schema1-renamed-[UNIQUE_NAME].myvolume' does not exist. === Redeploy is a no-op >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/volumes/move_from_renamed_schema/script b/acceptance/bundle/resources/volumes/move_from_renamed_schema/script index 1f4b6782023..cdfcc6ad6ab 100644 --- a/acceptance/bundle/resources/volumes/move_from_renamed_schema/script +++ b/acceptance/bundle/resources/volumes/move_from_renamed_schema/script @@ -27,8 +27,8 @@ trace print_requests.py //unity-catalog title "The volume is in the second schema" trace $CLI volumes read "$NEW_VOLUME" | jq '{full_name}' -title "The old volume moved with the renamed schema. It should have been deleted" -trace $CLI volumes read "$RENAMED_VOLUME" | jq '{full_name, volume_id}' +title "The old volume, moved with the renamed schema, was deleted there" +trace musterr $CLI volumes read "$RENAMED_VOLUME" title "Redeploy is a no-op" trace $CLI bundle plan diff --git a/acceptance/bundle/resources/volumes/move_from_renamed_schema/test.toml b/acceptance/bundle/resources/volumes/move_from_renamed_schema/test.toml index 4bf1280df97..4272a80cf61 100644 --- a/acceptance/bundle/resources/volumes/move_from_renamed_schema/test.toml +++ b/acceptance/bundle/resources/volumes/move_from_renamed_schema/test.toml @@ -1,4 +1,3 @@ -Badness = "The schema's rename moves the volume, but the recreate deletes the volume at its recorded path, which no longer exists. The volume moved with the schema is left behind untracked." Cloud = true Ignore = [ diff --git a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/output.txt b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/output.txt index 1c07fe27277..a5290335c5c 100644 --- a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/output.txt +++ b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/output.txt @@ -57,6 +57,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } }, "resources.volumes.volume1": { + "id": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", "depends_on": [ { "node": "resources.schemas.schema1", @@ -138,7 +139,7 @@ Resources: 1 created, 1 changed, 1 deleted, 0 unchanged } { "method": "DELETE", - "path": "/api/2.1/unity-catalog/volumes/main.test-schema-[UNIQUE_NAME].myvolume" + "path": "/api/2.1/unity-catalog/volumes/main.test-schema-renamed-[UNIQUE_NAME].myvolume" } { "method": "POST", diff --git a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/test.toml b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/test.toml index e9adadf175d..263d5495591 100644 --- a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/test.toml +++ b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/test.toml @@ -1,4 +1,3 @@ -Badness = "The schema's rename moves the volume, but the recreate deletes the volume at its recorded path, which no longer exists, and then creates it under the name the moved volume still has. The test server accepts that create, overwriting the moved volume; a real workspace would reject it because the name is taken." # Local only: an EXTERNAL volume needs an external location, which the test workspaces do not have. # volumes/move_from_renamed_schema covers the same recreate on a real workspace. diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index af154ab8740..1f099b32c2f 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -124,8 +124,9 @@ func LoadPlanFromFile(path string) (*Plan, error) { } type PlanEntry struct { - // ID is set when the resource was found at another ID than the one in state: a rename - // moved it there (see findMoved). Applying the entry addresses the resource at ID. + // ID is set when the resource is at another ID than the one in state: a rename moved it there + // (see findMoved), or a parent renamed in this plan moves it before it is recreated (see + // addParentMoves). Applying the entry addresses the resource at ID. ID string `json:"id,omitempty"` DependsOn []DependsOnEntry `json:"depends_on,omitempty"` Action ActionType `json:"action,omitempty"` diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index ce36c7e3c56..fa8fae30b1a 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -419,6 +419,10 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return nil, errors.New("planning failed") } + if err := b.addParentMoves(plan); err != nil { + return nil, err + } + for _, entry := range plan.Plan { if entry.Action == deployplan.Skip { entry.NewState = nil @@ -750,6 +754,91 @@ func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan. return change, nil } +// addParentMoves handles a recreate of a resource whose parent, as recorded in its state's +// depends_on, is renamed in the same plan. The backend moves the resource along with the parent, +// so the recreate has to delete it at the ID the rename gives it, after the rename: the entry's ID +// is set to that ID and the entry depends on the parent. The config may no longer reference the +// parent (the resource moves to another one), which is why the parent comes from the state. +func (b *DeploymentBundle) addParentMoves(plan *deployplan.Plan) error { + for resourceKey, entry := range plan.Plan { + if entry.Action != deployplan.Recreate || entry.ID != "" { + continue + } + adapter, err := b.getAdapterForKey(resourceKey) + if err != nil { + return err + } + if !adapter.HasIDFromState() { + continue + } + dbentry, ok := b.StateDB.GetResourceEntry(resourceKey) + if !ok { + continue + } + state, err := parseState(adapter.StateType(), dbentry.State) + if err != nil { + return fmt.Errorf("%s: interpreting state: %w", resourceKey, err) + } + + var parents []deployplan.DependsOnEntry + for _, dep := range dbentry.DependsOn { + rename, err := b.dependencyRename(dep.Label) + if err != nil { + return fmt.Errorf("%s: %w", resourceKey, err) + } + if rename == nil { + continue + } + for pathString, ch := range entry.Changes { + if !structdiff.IsEqual(ch.Old, rename.Old) { + continue + } + path, err := structpath.ParsePath(pathString) + if err != nil { + return err + } + if err := structaccess.Set(state, path, rename.New); err != nil { + return fmt.Errorf("%s: setting %s: %w", resourceKey, pathString, err) + } + parents = append(parents, dep) + } + } + if len(parents) == 0 { + continue + } + + id, err := adapter.IDFromState(state) + if err != nil { + return fmt.Errorf("%s: %w", resourceKey, err) + } + if id == dbentry.ID { + continue + } + entry.ID = id + for _, parent := range parents { + if !slices.ContainsFunc(entry.DependsOn, func(d deployplan.DependsOnEntry) bool { return d.Node == parent.Node }) { + entry.DependsOn = append(entry.DependsOn, parent) + } + } + } + return nil +} + +// dependencyRename returns the rename of the field a depends_on label references, nil if the label +// is not a pure reference or that field is not renamed in this plan. +func (b *DeploymentBundle) dependencyRename(label string) (*deployplan.ChangeDesc, error) { + ref, ok := dynvar.NewRef(dyn.V(label)) + if !ok || !ref.IsPure() { + return nil, nil + } + targetPath, err := structpath.ParsePath(ref.References()[0]) + if err != nil { + return nil, fmt.Errorf("cannot parse reference %q: %w", label, err) + } + targetKey, targetField := splitResourcePath(targetPath) + return b.renameChange(targetKey, targetField.String()) +} + // findMoved looks for a resource missing from its recorded ID at the ID its config gives it, where a // rename can have moved it: the resource or its parent renamed out of band, or by a deploy that // stopped before re-keying it. What is found there is taken only if its internal ID matches the one diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 42101800f21..887a046f167 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/structs/structdiff" @@ -876,3 +877,79 @@ func TestInternalIDChangesAreSkipped(t *testing.T) { } } } + +func TestAddParentMoves(t *testing.T) { + const ( + schemaKey = "resources.schemas.schema1" + volumeKey = "resources.volumes.volume1" + ) + schema1Dep := deployplan.DependsOnEntry{Node: schemaKey, Label: "${resources.schemas.schema1.name}"} + schema2Dep := deployplan.DependsOnEntry{Node: "resources.schemas.schema2", Label: "${resources.schemas.schema2.name}"} + + tests := []struct { + name string + schemaRenamed bool + volumeChanges deployplan.Changes + wantID string + wantDependsOn []deployplan.DependsOnEntry + }{ + { + name: "moved to another schema", + schemaRenamed: true, + volumeChanges: deployplan.Changes{"schema_name": {Action: deployplan.Recreate, Old: "s1", New: "s2"}}, + wantID: "main.s1-renamed.vol", + wantDependsOn: []deployplan.DependsOnEntry{schema2Dep, schema1Dep}, + }, + { + name: "recreated in the renamed schema", + schemaRenamed: true, + volumeChanges: deployplan.Changes{ + "schema_name": {Action: deployplan.UpdateWithID, Reason: deployplan.ReasonParentRenamed, Old: "s1", New: "s1-renamed"}, + "volume_type": {Action: deployplan.Recreate, Old: "MANAGED", New: "EXTERNAL"}, + }, + wantID: "main.s1-renamed.vol", + wantDependsOn: []deployplan.DependsOnEntry{schema2Dep, schema1Dep}, + }, + { + name: "schema not renamed", + schemaRenamed: false, + volumeChanges: deployplan.Changes{"schema_name": {Action: deployplan.Recreate, Old: "s1", New: "s2"}}, + wantID: "", + wantDependsOn: []deployplan.DependsOnEntry{schema2Dep}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) + + schemaEntry := &deployplan.PlanEntry{Action: deployplan.Skip} + if tt.schemaRenamed { + schemaEntry = &deployplan.PlanEntry{ + Action: deployplan.UpdateWithID, + Changes: deployplan.Changes{"name": {Action: deployplan.UpdateWithID, Old: "s1", New: "s1-renamed"}}, + } + } + plan := deployplan.NewPlanDirect() + plan.Plan[schemaKey] = schemaEntry + plan.Plan[volumeKey] = &deployplan.PlanEntry{ + Action: deployplan.Recreate, + DependsOn: []deployplan.DependsOnEntry{schema2Dep}, + Changes: tt.volumeChanges, + } + + b := &DeploymentBundle{Adapters: adapters, Plan: plan} + b.StateDB.OpenWithData("state.json", dstate.Database{State: map[string]dstate.ResourceEntry{ + volumeKey: { + ID: "main.s1.vol", + State: []byte(`{"catalog_name": "main", "schema_name": "s1", "name": "vol", "volume_type": "MANAGED"}`), + DependsOn: []deployplan.DependsOnEntry{schema1Dep}, + }, + }}) + + require.NoError(t, b.addParentMoves(plan)) + assert.Equal(t, tt.wantID, plan.Plan[volumeKey].ID) + assert.Equal(t, tt.wantDependsOn, plan.Plan[volumeKey].DependsOn) + }) + } +} From d2b71287021b8c6335e8947c43331bdefd6b83d5 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 16:37:43 +0100 Subject: [PATCH 40/75] acc: also deploy the key rename tests from a saved plan (READPLAN) The plan's state_only re-keys, found IDs and added depends_on must survive serialization. Co-authored-by: Isaac --- acceptance/bundle/resources/catalogs/rename/out.test.toml | 1 + acceptance/bundle/resources/catalogs/rename/script | 4 ++-- acceptance/bundle/resources/catalogs/rename/test.toml | 7 +++++++ .../resources/schemas/rename_with_volume/out.test.toml | 1 + .../bundle/resources/schemas/rename_with_volume/script | 4 ++-- .../bundle/resources/schemas/rename_with_volume/test.toml | 7 +++++++ .../volumes/move_from_renamed_schema/out.test.toml | 1 + .../resources/volumes/move_from_renamed_schema/script | 4 ++-- .../resources/volumes/move_from_renamed_schema/test.toml | 7 +++++++ .../volumes/rename_catalog_and_schema/out.test.toml | 1 + .../resources/volumes/rename_catalog_and_schema/script | 4 ++-- .../resources/volumes/rename_catalog_and_schema/test.toml | 7 +++++++ .../resources/volumes/rename_out_of_band/out.test.toml | 1 + .../bundle/resources/volumes/rename_out_of_band/script | 4 ++-- .../bundle/resources/volumes/rename_out_of_band/test.toml | 7 +++++++ 15 files changed, 50 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename/out.test.toml b/acceptance/bundle/resources/catalogs/rename/out.test.toml index 0f167effef1..7a62f726f25 100644 --- a/acceptance/bundle/resources/catalogs/rename/out.test.toml +++ b/acceptance/bundle/resources/catalogs/rename/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script index e84ef070fc8..fea2b758bf0 100644 --- a/acceptance/bundle/resources/catalogs/rename/script +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -22,10 +22,10 @@ trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ title "UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them" # Drop the remote_state fields that differ between clouds or from the test server. -trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location +trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location title "Deploy renames the catalog. UC moves the schema and the volume with it, so they get no request" -trace $CLI bundle deploy +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) trace print_requests.py //unity-catalog title "The old catalog is gone and the schema and the volume are under the new one, with their original ids\n" diff --git a/acceptance/bundle/resources/catalogs/rename/test.toml b/acceptance/bundle/resources/catalogs/rename/test.toml index 4272a80cf61..ebd892fd9df 100644 --- a/acceptance/bundle/resources/catalogs/rename/test.toml +++ b/acceptance/bundle/resources/catalogs/rename/test.toml @@ -1,7 +1,14 @@ Cloud = true +EnvMatrix.READPLAN = ["", "1"] Ignore = [ ".databricks", ".gitignore", "databricks.yml", ] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/out.test.toml b/acceptance/bundle/resources/schemas/rename_with_volume/out.test.toml index 0f167effef1..7a62f726f25 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/out.test.toml +++ b/acceptance/bundle/resources/schemas/rename_with_volume/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/script b/acceptance/bundle/resources/schemas/rename_with_volume/script index fd5830bf18a..d5a3a95554c 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/script +++ b/acceptance/bundle/resources/schemas/rename_with_volume/script @@ -22,10 +22,10 @@ trace update_file.py databricks.yml test-schema- test-schema-renamed- title "UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it" # Drop the remote_state fields that differ between clouds or from the test server. -trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location title "Deploy renames the schema. UC moves the volume with it, so the volume gets no request" -trace $CLI bundle deploy +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) trace print_requests.py //unity-catalog title "The old schema is gone and the volume is under the new one, with its original volume_id" diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/test.toml b/acceptance/bundle/resources/schemas/rename_with_volume/test.toml index 4272a80cf61..ebd892fd9df 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/test.toml +++ b/acceptance/bundle/resources/schemas/rename_with_volume/test.toml @@ -1,7 +1,14 @@ Cloud = true +EnvMatrix.READPLAN = ["", "1"] Ignore = [ ".databricks", ".gitignore", "databricks.yml", ] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' diff --git a/acceptance/bundle/resources/volumes/move_from_renamed_schema/out.test.toml b/acceptance/bundle/resources/volumes/move_from_renamed_schema/out.test.toml index 0f167effef1..7a62f726f25 100644 --- a/acceptance/bundle/resources/volumes/move_from_renamed_schema/out.test.toml +++ b/acceptance/bundle/resources/volumes/move_from_renamed_schema/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/volumes/move_from_renamed_schema/script b/acceptance/bundle/resources/volumes/move_from_renamed_schema/script index cdfcc6ad6ab..5d038da5e21 100644 --- a/acceptance/bundle/resources/volumes/move_from_renamed_schema/script +++ b/acceptance/bundle/resources/volumes/move_from_renamed_schema/script @@ -20,8 +20,8 @@ title "Rename the first schema and move the volume to the second one in one depl trace update_file.py databricks.yml test-schema1- test-schema1-renamed- trace update_file.py databricks.yml schemas.schema1.name schemas.schema2.name # Drop the remote_state fields that differ between clouds or from the test server. -trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location -trace $CLI bundle deploy --auto-approve +trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) --auto-approve trace print_requests.py //unity-catalog title "The volume is in the second schema" diff --git a/acceptance/bundle/resources/volumes/move_from_renamed_schema/test.toml b/acceptance/bundle/resources/volumes/move_from_renamed_schema/test.toml index 4272a80cf61..ebd892fd9df 100644 --- a/acceptance/bundle/resources/volumes/move_from_renamed_schema/test.toml +++ b/acceptance/bundle/resources/volumes/move_from_renamed_schema/test.toml @@ -1,7 +1,14 @@ Cloud = true +EnvMatrix.READPLAN = ["", "1"] Ignore = [ ".databricks", ".gitignore", "databricks.yml", ] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/out.test.toml b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/out.test.toml index 0f167effef1..7a62f726f25 100644 --- a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/out.test.toml +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/script b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/script index 7b2ea667360..2f0b71af10b 100644 --- a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/script +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/script @@ -22,10 +22,10 @@ trace update_file.py databricks.yml "name: myschema" "name: myschema2" title "Both of the volume's parents are renamed, so both its catalog_name and schema_name follow them" # Drop the remote_state fields that differ between clouds or from the test server. -trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location +trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location title "Deploy renames the catalog, then the schema under it. UC moves the volume both times, so it gets no request" -trace $CLI bundle deploy +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) trace print_requests.py //unity-catalog title "The schema and the volume are under the new names, with their original ids" diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/test.toml b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/test.toml index 4272a80cf61..ebd892fd9df 100644 --- a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/test.toml +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/test.toml @@ -1,7 +1,14 @@ Cloud = true +EnvMatrix.READPLAN = ["", "1"] Ignore = [ ".databricks", ".gitignore", "databricks.yml", ] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/out.test.toml b/acceptance/bundle/resources/volumes/rename_out_of_band/out.test.toml index 0f167effef1..7a62f726f25 100644 --- a/acceptance/bundle/resources/volumes/rename_out_of_band/out.test.toml +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/script b/acceptance/bundle/resources/volumes/rename_out_of_band/script index c16fde22dbc..fc647a5b8ea 100644 --- a/acceptance/bundle/resources/volumes/rename_out_of_band/script +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/script @@ -12,7 +12,7 @@ trap cleanup EXIT # Drop the remote_state fields that differ between clouds or from the test server. plan() { - trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location + trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location } title "Deploy a schema with a volume in it" @@ -32,7 +32,7 @@ trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" plan title "Deploy only records the new name in the state" -trace $CLI bundle deploy +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) trace print_requests.py //unity-catalog trace $CLI volumes read "$SCHEMA.myvolume2" | jq '{full_name, volume_id}' trace $CLI bundle summary --output json | jq '.resources.volumes.volume1.id' diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/test.toml b/acceptance/bundle/resources/volumes/rename_out_of_band/test.toml index 4272a80cf61..ebd892fd9df 100644 --- a/acceptance/bundle/resources/volumes/rename_out_of_band/test.toml +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/test.toml @@ -1,7 +1,14 @@ Cloud = true +EnvMatrix.READPLAN = ["", "1"] Ignore = [ ".databricks", ".gitignore", "databricks.yml", ] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' From 7ab54eb24759ba175cd18dcd3f32b3603637c75a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 17:27:07 +0100 Subject: [PATCH 41/75] acc: regenerate the PR's goldens after rebasing on main Numbered UUIDs from main (#7021); restores the volume_id skip in change-name and the recorded schema_id in user_agent/simple, which the rebase dropped. Co-authored-by: Isaac --- .../schemas/recreate_interrupted/output.txt | 10 +++++----- .../schemas/rename_interrupted/output.txt | 12 ++++++------ .../volumes/change-name/out.plan.json | 6 ++++++ .../move_from_renamed_schema/output.txt | 12 ++++++------ .../recreate_in_renamed_schema/output.txt | 18 +++++++++--------- .../user_agent/simple/out.requests.deploy.json | 3 ++- 6 files changed, 34 insertions(+), 27 deletions(-) diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt b/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt index 3be652c4773..b8595d0fa84 100644 --- a/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt @@ -62,10 +62,10 @@ Exit code: [KILLED] }, "enable_predictive_optimization": "INHERIT", "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", - "metastore_id": "[UUID]", + "metastore_id": "[UUID][0]", "name": "test-schema-renamed-[UNIQUE_NAME]", "owner": "[USERNAME]", - "schema_id": "[UUID]", + "schema_id": "[UUID][1]", "storage_root": "s3://mybucket/b", "updated_by": "[USERNAME]" }, @@ -73,8 +73,8 @@ Exit code: [KILLED] "schema_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } }, @@ -132,7 +132,7 @@ Resources: 1 created, 0 changed, 0 deleted, 1 unchanged >>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume { "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", - "volume_id": "[UUID]" + "volume_id": "[UUID][2]" } === Redeploy is a no-op diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt index 74f616d68b7..27039e2150b 100644 --- a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt +++ b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt @@ -54,10 +54,10 @@ Exit code: [KILLED] }, "enable_predictive_optimization": "INHERIT", "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", - "metastore_id": "[UUID]", + "metastore_id": "[UUID][0]", "name": "test-schema-renamed-[UNIQUE_NAME]", "owner": "[USERNAME]", - "schema_id": "[UUID]", + "schema_id": "[UUID][1]", "updated_by": "[USERNAME]" }, "changes": { @@ -74,8 +74,8 @@ Exit code: [KILLED] "schema_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } }, @@ -105,7 +105,7 @@ Exit code: [KILLED] "name": "myvolume", "owner": "[USERNAME]", "schema_name": "test-schema-renamed-[UNIQUE_NAME]", - "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[VOLUME_ID]", + "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID][0]/volumes/[VOLUME_ID]", "volume_id": "[VOLUME_ID]", "volume_type": "MANAGED" }, @@ -126,7 +126,7 @@ Exit code: [KILLED] "storage_location": { "action": "skip", "reason": "immutable_backend_value", - "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[VOLUME_ID]" + "remote": "s3://[METASTORE_NAME]/metastore/[UUID][0]/volumes/[VOLUME_ID]" }, "volume_id": { "action": "skip", diff --git a/acceptance/bundle/resources/volumes/change-name/out.plan.json b/acceptance/bundle/resources/volumes/change-name/out.plan.json index 47d55af7ad3..cc6403736d3 100644 --- a/acceptance/bundle/resources/volumes/change-name/out.plan.json +++ b/acceptance/bundle/resources/volumes/change-name/out.plan.json @@ -41,6 +41,12 @@ "action": "skip", "reason": "immutable_backend_value", "remote": "s3://[METASTORE_NAME]/metastore/[UUID][1]/volumes/[UUID][2]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][2]", + "remote": "[UUID][2]" } } } diff --git a/acceptance/bundle/resources/volumes/move_from_renamed_schema/output.txt b/acceptance/bundle/resources/volumes/move_from_renamed_schema/output.txt index 001063266cf..ccee5240bef 100644 --- a/acceptance/bundle/resources/volumes/move_from_renamed_schema/output.txt +++ b/acceptance/bundle/resources/volumes/move_from_renamed_schema/output.txt @@ -32,7 +32,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "full_name": "main.test-schema1-[UNIQUE_NAME]", "name": "test-schema1-[UNIQUE_NAME]", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[UUID][0]" }, "changes": { "name": { @@ -45,8 +45,8 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "schema_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][0]", + "remote": "[UUID][0]" } } }, @@ -61,14 +61,14 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "full_name": "main.test-schema2-[UNIQUE_NAME]", "name": "test-schema2-[UNIQUE_NAME]", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[UUID][1]" }, "changes": { "schema_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } }, diff --git a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/output.txt b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/output.txt index a5290335c5c..82200b34596 100644 --- a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/output.txt +++ b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/output.txt @@ -34,10 +34,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged }, "enable_predictive_optimization": "INHERIT", "full_name": "main.test-schema-[UNIQUE_NAME]", - "metastore_id": "[UUID]", + "metastore_id": "[UUID][0]", "name": "test-schema-[UNIQUE_NAME]", "owner": "[USERNAME]", - "schema_id": "[UUID]", + "schema_id": "[UUID][1]", "updated_by": "[USERNAME]" }, "changes": { @@ -51,8 +51,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "schema_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } }, @@ -80,8 +80,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "name": "myvolume", "owner": "[USERNAME]", "schema_name": "test-schema-[UNIQUE_NAME]", - "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", - "volume_id": "[UUID]", + "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID][0]/volumes/[UUID][2]", + "volume_id": "[UUID][2]", "volume_type": "MANAGED" }, "changes": { @@ -95,13 +95,13 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "storage_location": { "action": "skip", "reason": "immutable_backend_value", - "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" + "remote": "s3://[METASTORE_NAME]/metastore/[UUID][0]/volumes/[UUID][2]" }, "volume_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][2]", + "remote": "[UUID][2]" }, "volume_type": { "action": "recreate", diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.json index 8479974039e..028011cb20c 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.json @@ -249,7 +249,8 @@ "__id__": "mycatalog.myschema", "state": { "catalog_name": "mycatalog", - "name": "myschema" + "name": "myschema", + "schema_id": "[UUID][4]" } } } From e268355ae2d9059fc5311589082d6d7cfd90426d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 17:50:45 +0100 Subject: [PATCH 42/75] acc: re-key a mixed-case volume under its renamed schema; testserver: case-insensitive volume lookups On AWS the re-keyed ID keeps the config's casing (main.s.MyVolume) and UC still reads the volume at it, so the next plan is a no-op. The test server now looks a volume's full name up case-insensitively too, instead of planning a create. Co-authored-by: Isaac --- .../volumes/rename_case_only/output.txt | 26 +++++++++++++++++++ .../resources/volumes/rename_case_only/script | 7 +++++ libs/testserver/handlers.go | 9 ++++--- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/volumes/rename_case_only/output.txt b/acceptance/bundle/resources/volumes/rename_case_only/output.txt index d0d42aa4476..3c1b9113c16 100644 --- a/acceptance/bundle/resources/volumes/rename_case_only/output.txt +++ b/acceptance/bundle/resources/volumes/rename_case_only/output.txt @@ -107,6 +107,32 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged +=== Rename the schema. The volume is re-keyed without a request. Its recorded ID keeps the config's casing, which UC reads case-insensitively +>>> update_file.py databricks.yml test_schema_ test_schema_renamed_ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test_schema_[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test_schema_renamed_[UNIQUE_NAME]" + } +} + +>>> [CLI] bundle summary --output json +"main.test_schema_renamed_[UNIQUE_NAME].MyVolume" + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.schemas.schema1 diff --git a/acceptance/bundle/resources/volumes/rename_case_only/script b/acceptance/bundle/resources/volumes/rename_case_only/script index ebaf4e89b14..21cea011eaf 100644 --- a/acceptance/bundle/resources/volumes/rename_case_only/script +++ b/acceptance/bundle/resources/volumes/rename_case_only/script @@ -25,3 +25,10 @@ trace $CLI volumes read "$VOLUME" | jq '{full_name, name, volume_id}' title "Redeploy" trace $CLI bundle plan + +title "Rename the schema. The volume is re-keyed without a request. Its recorded ID keeps the config's casing, which UC reads case-insensitively" +trace update_file.py databricks.yml test_schema_ test_schema_renamed_ +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI bundle summary --output json | jq '.resources.volumes.volume1.id' +trace $CLI bundle plan diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 8d7708af5af..f840b33cf9c 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -986,10 +986,11 @@ func AddDefaultHandlers(server *Server) { return MapDelete(req.Workspace, req.Workspace.ModelProviderServices, req.Vars["name"]) }) - // Volumes: + // Volumes: stored under their lowercased full name; UC looks a full name up case-insensitively + // (checked on AWS: a volume re-keyed as main.s.MyVolume is still read at that path). server.Handle("GET", "/api/2.1/unity-catalog/volumes/{full_name}", func(req Request) any { - return MapGetUC(req.Workspace, req.Workspace.Volumes, req.Vars["full_name"], "Volume") + return MapGetUC(req.Workspace, req.Workspace.Volumes, strings.ToLower(req.Vars["full_name"]), "Volume") }) server.Handle("POST", "/api/2.1/unity-catalog/volumes", func(req Request) any { @@ -997,11 +998,11 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("PATCH", "/api/2.1/unity-catalog/volumes/{full_name}", func(req Request) any { - return req.Workspace.VolumesUpdate(req, req.Vars["full_name"]) + return req.Workspace.VolumesUpdate(req, strings.ToLower(req.Vars["full_name"])) }) server.Handle("DELETE", "/api/2.1/unity-catalog/volumes/{full_name}", func(req Request) any { - return MapDelete(req.Workspace, req.Workspace.Volumes, req.Vars["full_name"]) + return MapDelete(req.Workspace, req.Workspace.Volumes, strings.ToLower(req.Vars["full_name"])) }) // Repos: From dd06627bbb21fc0e015460e48720b340b7dce87c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 18:06:40 +0100 Subject: [PATCH 43/75] direct: follow a parent rename only with IDFromState (review B4) Without IDFromState the update_id would reach DoUpdateWithID with the stale ID. Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index fa8fae30b1a..f7e6618c73c 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -897,12 +897,13 @@ func hasUnresolvedIDField(adapter *dresources.Adapter, sv *structvar.StructVar) // followsParentRename reports whether a change to a provided ID field only follows parentChange, // the rename of the parent field it references (see parentRenames). UC moves the children of a // renamed catalog or schema along with it, so recreating such a child fails with "already -// exists"; it is planned as an update_id instead, which addresses it under the new parent. +// exists"; it is planned as an update_id instead, which addresses it under the new parent. That +// needs IDFromState: with no other change, the update_id only re-keys the state (StateOnly). // // The old values must match too: a child moved to the renamed parent from somewhere else // still has to be recreated. func followsParentRename(adapter *dresources.Adapter, parentChange, ch *deployplan.ChangeDesc) bool { - return parentChange != nil && adapter.HasDoUpdateWithID() && structdiff.IsEqual(ch.Old, parentChange.Old) + return parentChange != nil && adapter.HasDoUpdateWithID() && adapter.HasIDFromState() && structdiff.IsEqual(ch.Old, parentChange.Old) } // shouldSkipNormalized skips a change that is a false diff caused by UC API From 80e8471c9b316a8897a8db1f875502c1cec3d1d5 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 18:07:00 +0100 Subject: [PATCH 44/75] direct: check generated ID fields for unresolved references too (review B5) Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index f7e6618c73c..5f29f5a2e9f 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -878,17 +878,18 @@ func findMoved(ctx context.Context, adapter *dresources.Adapter, sv *structvar.S // hasUnresolvedIDField reports whether a field the ID is built from is still an unresolved reference. func hasUnresolvedIDField(adapter *dresources.Adapter, sv *structvar.StructVar) bool { - cfg := adapter.ResourceConfig() for pathString := range sv.Refs { path, err := structpath.ParsePath(pathString) if err != nil { return true } - if _, ok := findMatchingRule(path, cfg.ProvidedIDFields); ok { - return true - } - if _, ok := findMatchingRule(path, cfg.UpdatableIDFields); ok { - return true + for _, cfg := range []*dresources.ResourceLifecycleConfig{adapter.ResourceConfig(), adapter.GeneratedResourceConfig()} { + if _, ok := findMatchingRule(path, cfg.ProvidedIDFields); ok { + return true + } + if _, ok := findMatchingRule(path, cfg.UpdatableIDFields); ok { + return true + } } } return false From 908b0d7ae93b455b8f2b2f18e731f31a02bf3c62 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 18:07:02 +0100 Subject: [PATCH 45/75] direct: report the ID that was read when a moved resource fails to load (review B6) Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 5f29f5a2e9f..9d39393f4ef 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -1,6 +1,7 @@ package direct import ( + "cmp" "context" "encoding/json" "errors" @@ -353,7 +354,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks if remoteState != nil { remoteStateComparable, err = adapter.RemapState(remoteState) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: interpreting remote state id=%q: %w", errorPrefix, dbentry.ID, err)) + logdiag.LogError(ctx, fmt.Errorf("%s: interpreting remote state id=%q: %w", errorPrefix, cmp.Or(entry.ID, dbentry.ID), err)) return false } @@ -361,7 +362,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks // is a hash on all three sides of the diff (saved, local, remote) remoteStateComparable, err = dresources.CompactState(adapter.ResourceConfig(), remoteStateComparable) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: compacting remote state id=%q: %w", errorPrefix, dbentry.ID, err)) + logdiag.LogError(ctx, fmt.Errorf("%s: compacting remote state id=%q: %w", errorPrefix, cmp.Or(entry.ID, dbentry.ID), err)) return false } From 04a0154e652ee50484c5a033f8d90a2c1af441b8 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 18:07:05 +0100 Subject: [PATCH 46/75] direct: collect parent renames only for types that can follow one (review B7) Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 9d39393f4ef..44e4109d362 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -264,11 +264,15 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return true } - // Collected before resolveReferences, which drops the references it resolves. - parentRenames, err := b.parentRenames(resourceKey) - if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) - return false + // Collected before resolveReferences, which drops the references it resolves. Only a resource + // that can follow a rename needs them (see followsParentRename). + var parentRenames map[string]*deployplan.ChangeDesc + if adapter.HasDoUpdateWithID() { + parentRenames, err = b.parentRenames(resourceKey) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } } // Process all references in the resource using Refs map From 5804bafd2f91440e17892917ae10dffb5b7ba592 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 18:07:18 +0100 Subject: [PATCH 47/75] dresources: show the new state types in the IResource examples (review B8) Co-authored-by: Isaac --- bundle/direct/dresources/adapter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 64c7fc22d8d..898eb90dbee 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -75,7 +75,7 @@ type IResource interface { // DoCreate creates a new resource from the newState. Returns id of the resource and optionally remote state. // If remote state is available as part of the operation, return it; otherwise return nil. - // Example: func (r *ResourceVolume) DoCreate(ctx context.Context, newState *catalog.CreateVolumeRequestContent) (string, *catalog.VolumeInfo, error) + // Example: func (r *ResourceVolume) DoCreate(ctx context.Context, newState *VolumeState) (string, *catalog.VolumeInfo, error) DoCreate(ctx context.Context, newState any) (id string, remoteState any, e error) // [Optional] IsEmptyState reports that newState describes no resource at all: the planner @@ -86,7 +86,7 @@ type IResource interface { // [Optional] DoUpdate updates the resource. ID must not change as a result of this operation. Returns optionally remote state. // If remote state is available as part of the operation, return it; otherwise return nil. - // Example: func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, newState *catalog.CreateSchema, entry *PlanEntry) (*catalog.SchemaInfo, error) + // Example: func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, newState *SchemaState, entry *PlanEntry) (*catalog.SchemaInfo, error) DoUpdate(ctx context.Context, id string, newState any, entry *PlanEntry) (remoteState any, e error) // [Optional] DoUpdateWithID performs an update that may result in resource having a new ID. Returns new id and optionally remote state. From a248efca90b92ed2f0c9b6fdbfc4260d5f75e280 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 18:11:10 +0100 Subject: [PATCH 48/75] acc: restore the internal_id skips main's UUID renumbering dropped from four goldens Co-authored-by: Isaac --- .../bundle/migrate/command/basic/out.plan_update.json | 5 +++++ .../grants/schemas/duplicate_principals/out.plan.json | 8 ++++++++ .../grants/schemas/out_of_band_principal/out.plan.json | 8 ++++++++ .../grants/schemas/out_of_band_principal/out.plan2.json | 8 ++++++++ 4 files changed, 29 insertions(+) diff --git a/acceptance/bundle/migrate/command/basic/out.plan_update.json b/acceptance/bundle/migrate/command/basic/out.plan_update.json index dc06b6c6a3c..4d7b49372d5 100644 --- a/acceptance/bundle/migrate/command/basic/out.plan_update.json +++ b/acceptance/bundle/migrate/command/basic/out.plan_update.json @@ -240,6 +240,11 @@ "action": "skip", "reason": "immutable_backend_value", "remote": "s3://[METASTORE_NAME]/metastore/[UUID][1]/volumes/[UUID][2]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "remote": "[UUID][2]" } } } diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.json b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.json index 885884210a3..48112060980 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.json +++ b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.json @@ -18,6 +18,14 @@ "name": "schema_dup_grants_[UNIQUE_NAME]", "owner": "[USERNAME]", "schema_id": "[UUID][2]" + }, + "changes": { + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][2]", + "remote": "[UUID][2]" + } } }, "resources.schemas.apps_schema.grants": { diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.json b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.json index dfb10ea9b05..2ddd1b9d54f 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.json +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.json @@ -18,6 +18,14 @@ "name": "schema_out_of_band_principal_[UNIQUE_NAME]", "owner": "[USERNAME]", "schema_id": "[UUID][2]" + }, + "changes": { + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][2]", + "remote": "[UUID][2]" + } } }, "resources.schemas.grants_schema.grants": { diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.json b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.json index 78b0aa501c0..53aa597f678 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.json +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.json @@ -18,6 +18,14 @@ "name": "schema_out_of_band_principal_[UNIQUE_NAME]", "owner": "[USERNAME]", "schema_id": "[UUID][2]" + }, + "changes": { + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][2]", + "remote": "[UUID][2]" + } } }, "resources.schemas.grants_schema.grants": { From 231bf356da0411b2f023d8500c1e72adea514221 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 18:25:46 +0100 Subject: [PATCH 49/75] acc: a state without schema_id/volume_id cannot follow an out-of-band rename (review A3, Badness); testserver: reject a taken schema name Recorded on AWS: the schema's create fails with SCHEMA_ALREADY_EXISTS. The test server now rejects it the same way instead of creating it again. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 13 ++ .../out.test.toml | 2 + .../rename_out_of_band_old_state/output.txt | 174 ++++++++++++++++++ .../rename_out_of_band_old_state/script | 52 ++++++ .../rename_out_of_band_old_state/test.toml | 12 ++ libs/testserver/schemas.go | 10 + 6 files changed, 263 insertions(+) create mode 100644 acceptance/bundle/resources/schemas/rename_out_of_band_old_state/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/rename_out_of_band_old_state/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_out_of_band_old_state/output.txt create mode 100644 acceptance/bundle/resources/schemas/rename_out_of_band_old_state/script create mode 100644 acceptance/bundle/resources/schemas/rename_out_of_band_old_state/test.toml diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/databricks.yml.tmpl new file mode 100644 index 00000000000..2234fdf2e39 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/out.test.toml b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/out.test.toml new file mode 100644 index 00000000000..f68b4a89c36 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/output.txt b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/output.txt new file mode 100644 index 00000000000..a7ea1b1547c --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/output.txt @@ -0,0 +1,174 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Drop schema_id and volume_id from the state, as a CLI from before they were recorded wrote it +>>> jq .state | map_values(.state | {schema_id, volume_id} | with_entries(select(.value != null))) .databricks/bundle/default/resources.json +{ + "resources.schemas.schema1": {}, + "resources.volumes.volume1": {} +} + +=== Deploy with nothing to change +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-[UNIQUE_NAME]", + "name": "test-schema-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[SCHEMA_ID]" + }, + "changes": { + "schema_id": { + "action": "skip", + "reason": "internal_id", + "remote": "[SCHEMA_ID]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "skip", + "remote_state": { + "catalog_name": "main", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "main.test-schema-[UNIQUE_NAME].myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "test-schema-[UNIQUE_NAME]", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "volume_id": { + "action": "skip", + "reason": "internal_id", + "remote": "[VOLUME_ID]" + } + } + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog + +>>> jq .state | map_values(.state | {schema_id, volume_id} | with_entries(select(.value != null))) .databricks/bundle/default/resources.json +{ + "resources.schemas.schema1": {}, + "resources.volumes.volume1": {} +} + +=== Rename the schema out of band, and the config to match +>>> [CLI] schemas update main.test-schema-[UNIQUE_NAME] --new-name test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" +} + +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "test-schema-renamed-[UNIQUE_NAME]" + } + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "changes": { + "schema_name": { + "action": "recreate", + "reason": "id_field", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]" + } + } + } +} + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Error: cannot create resources.schemas.schema1: Schema 'test-schema-renamed-[UNIQUE_NAME]' already exists (400 SCHEMA_ALREADY_EXISTS) + +Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/schemas +HTTP Status: 400 Bad Request +API error_code: SCHEMA_ALREADY_EXISTS +API message: Schema 'test-schema-renamed-[UNIQUE_NAME]' already exists + +Error: cannot create resources.volumes.volume1: dependency failed: resources.schemas.schema1 + +Files: 2 uploaded, 0 deleted + +Exit code: 1 + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "test-schema-renamed-[UNIQUE_NAME]" + } +} + +=== Redeploy +>>> errcode [CLI] bundle plan +create schemas.schema1 +create volumes.volume1 + +Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 0 deleted diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/script b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/script new file mode 100644 index 00000000000..650a14efa24 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/script @@ -0,0 +1,52 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_SCHEMA="main.test-schema-${UNIQUE_NAME}" +NEW_SCHEMA="main.test-schema-renamed-${UNIQUE_NAME}" +STATE=.databricks/bundle/default/resources.json + +cleanup() { + trace $CLI bundle destroy --auto-approve + # Whatever the state lost track of. + errcode $CLI schemas delete "$NEW_SCHEMA" --force > LOG.cleanup 2>&1 + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +} + +recorded_ids() { + trace jq '.state | map_values(.state | {schema_id, volume_id} | with_entries(select(.value != null)))' "$STATE" +} + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "$OLD_SCHEMA" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "$OLD_SCHEMA.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Drop schema_id and volume_id from the state, as a CLI from before they were recorded wrote it" +jq 'del(.state["resources.schemas.schema1"].state.schema_id, .state["resources.volumes.volume1"].state.volume_id)' "$STATE" > tmp.json +mv tmp.json "$STATE" +recorded_ids + +title "Deploy with nothing to change" +plan +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +recorded_ids + +title "Rename the schema out of band, and the config to match" +trace $CLI schemas update "$OLD_SCHEMA" --new-name "test-schema-renamed-${UNIQUE_NAME}" | jq '{full_name, schema_id}' +print_requests.py //unity-catalog &> LOG.rename_requests +trace update_file.py databricks.yml test-schema- test-schema-renamed- +plan +trace errcode $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "Redeploy" +trace errcode $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/test.toml b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/test.toml new file mode 100644 index 00000000000..5e20d4c81ce --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/test.toml @@ -0,0 +1,12 @@ +Badness = "A state recorded without schema_id and volume_id (by a CLI from before they were recorded) never gets them, since a deploy with nothing to change writes no state. So the plan cannot find the schema and the volume renamed out of band, and creates them again." +Cloud = true + +# The test rewrites the deployment state by hand, which the recording service refuses because it +# no longer matches the deployment it has recorded, so run without deployment history. +EnvMatrix.DMS = [""] + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index 78fdd13903f..348deece67a 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -34,6 +34,16 @@ func (s *FakeWorkspace) SchemasCreate(req Request) Response { // Strip trailing slash to mimic UC API normalization behavior (see volumes.go). schema.StorageRoot = strings.TrimRight(schema.StorageRoot, "/") schema.FullName = schema.CatalogName + "." + schema.Name + + if _, exists := s.Schemas[schema.FullName]; exists { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{ + "error_code": "SCHEMA_ALREADY_EXISTS", + "message": fmt.Sprintf("Schema '%s' already exists", schema.Name), + }, + } + } schema.ForceSendFields = []string{"BrowseOnly"} schema.CatalogType = "MANAGED_CATALOG" schema.CreatedAt = nowMilli() From 894f9b0c46e5a837363a8fe3444cddbac666dcaf Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 19:49:06 +0100 Subject: [PATCH 50/75] acc: state_present deletes the replayed schema before creating it again The test server now rejects a taken schema name, as UC does. Co-authored-by: Isaac --- acceptance/bundle/state/state_present/script | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/acceptance/bundle/state/state_present/script b/acceptance/bundle/state/state_present/script index 22122d53b07..45afcfb30f0 100644 --- a/acceptance/bundle/state/state_present/script +++ b/acceptance/bundle/state/state_present/script @@ -26,6 +26,10 @@ echo "{\"serial\": 10, \"lineage\": \"$lineage\"}" > .databricks/bundle/default/ # serial 10 outranks the terraform state (serial 1), so direct wins; its (empty) state plans to # create the schema, and every schemas request carries the direct-engine User-Agent, not terraform. trace $CLI bundle plan +# The replayed terraform deployment created the schema, which the empty direct state does not know +# about; delete it so the create does not fail with SCHEMA_ALREADY_EXISTS. +$CLI schemas delete main.myschema +rm "$OUT_REQUESTS" trace $CLI bundle deploy trace print_requests.py //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'engine/direct' '!tf-provider' trace $CLI bundle destroy --auto-approve From 53a06c59675329e2113a1f193fa00c29c29ede2f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 19:51:55 +0100 Subject: [PATCH 51/75] direct: record a missing internal ID with a state-only update (review A3) A state recorded without schema_id/volume_id (by an older CLI, or by bind) never got them, since a skip writes no state, so the plan could not follow an out-of-band rename. An otherwise unchanged resource whose state lacks the ID is now planned as a state-only update that records it from the remote state. Co-authored-by: Isaac --- .../rename_out_of_band_old_state/output.txt | 135 ++++++++++++------ .../rename_out_of_band_old_state/script | 10 +- .../rename_out_of_band_old_state/test.toml | 1 - bundle/deployplan/plan.go | 3 + bundle/direct/apply.go | 13 ++ bundle/direct/bundle_plan.go | 27 ++++ 6 files changed, 142 insertions(+), 47 deletions(-) diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/output.txt b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/output.txt index a7ea1b1547c..604d46a7586 100644 --- a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/output.txt +++ b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/output.txt @@ -14,11 +14,18 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "resources.volumes.volume1": {} } -=== Deploy with nothing to change +=== A deploy with nothing else to change records them, with a state-only update >>> [CLI] bundle plan --output json { "resources.schemas.schema1": { - "action": "skip", + "action": "update", + "state_only": true, + "new_state": { + "value": { + "catalog_name": "main", + "name": "test-schema-[UNIQUE_NAME]" + } + }, "remote_state": { "browse_only": false, "catalog_name": "main", @@ -46,7 +53,16 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "label": "${resources.schemas.schema1.name}" } ], - "action": "skip", + "action": "update", + "state_only": true, + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, "remote_state": { "catalog_name": "main", "created_at": [UNIX_TIME_MILLIS][1], @@ -77,11 +93,15 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> jq .state | map_values(.state | {schema_id, volume_id} | with_entries(select(.value != null))) .databricks/bundle/default/resources.json { - "resources.schemas.schema1": {}, - "resources.volumes.volume1": {} + "resources.schemas.schema1": { + "schema_id": "[SCHEMA_ID]" + }, + "resources.volumes.volume1": { + "volume_id": "[VOLUME_ID]" + } } -=== Rename the schema out of band, and the config to match +=== Rename the schema out of band, and the config to match. The plan finds both by their ids >>> [CLI] schemas update main.test-schema-[UNIQUE_NAME] --new-name test-schema-renamed-[UNIQUE_NAME] { "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", @@ -93,30 +113,53 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> [CLI] bundle plan --output json { "resources.schemas.schema1": { - "action": "create", + "id": "main.test-schema-renamed-[UNIQUE_NAME]", + "action": "update_id", + "state_only": true, "new_state": { "value": { "catalog_name": "main", "name": "test-schema-renamed-[UNIQUE_NAME]" } }, + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "name": "test-schema-renamed-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[SCHEMA_ID]" + }, "changes": { "name": { - "action": "update_id", - "reason": "id_changes", + "action": "skip", + "reason": "remote_already_set", "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]" + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-renamed-[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]", + "remote": "[SCHEMA_ID]" } } }, "resources.volumes.volume1": { + "id": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", "depends_on": [ { "node": "resources.schemas.schema1", "label": "${resources.schemas.schema1.name}" } ], - "action": "create", + "action": "update_id", + "state_only": true, "new_state": { "value": { "catalog_name": "main", @@ -125,50 +168,60 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged "volume_type": "MANAGED" } }, + "remote_state": { + "catalog_name": "main", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, "changes": { "schema_name": { - "action": "recreate", - "reason": "id_field", + "action": "skip", + "reason": "remote_already_set", "old": "test-schema-[UNIQUE_NAME]", - "new": "test-schema-renamed-[UNIQUE_NAME]" + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-renamed-[UNIQUE_NAME]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" } } } } ->>> errcode [CLI] bundle deploy +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... -Error: cannot create resources.schemas.schema1: Schema 'test-schema-renamed-[UNIQUE_NAME]' already exists (400 SCHEMA_ALREADY_EXISTS) - -Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/schemas -HTTP Status: 400 Bad Request -API error_code: SCHEMA_ALREADY_EXISTS -API message: Schema 'test-schema-renamed-[UNIQUE_NAME]' already exists - -Error: cannot create resources.volumes.volume1: dependency failed: resources.schemas.schema1 - Files: 2 uploaded, 0 deleted - -Exit code: 1 +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //unity-catalog -{ - "method": "POST", - "path": "/api/2.1/unity-catalog/schemas", - "body": { - "catalog_name": "main", - "name": "test-schema-renamed-[UNIQUE_NAME]" - } -} - -=== Redeploy ->>> errcode [CLI] bundle plan -create schemas.schema1 -create volumes.volume1 -Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default -Destroy: 0 deleted +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/script b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/script index 650a14efa24..bd43dd908ba 100644 --- a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/script +++ b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/script @@ -34,19 +34,19 @@ jq 'del(.state["resources.schemas.schema1"].state.schema_id, .state["resources.v mv tmp.json "$STATE" recorded_ids -title "Deploy with nothing to change" +title "A deploy with nothing else to change records them, with a state-only update" plan trace $CLI bundle deploy trace print_requests.py //unity-catalog recorded_ids -title "Rename the schema out of band, and the config to match" +title "Rename the schema out of band, and the config to match. The plan finds both by their ids" trace $CLI schemas update "$OLD_SCHEMA" --new-name "test-schema-renamed-${UNIQUE_NAME}" | jq '{full_name, schema_id}' print_requests.py //unity-catalog &> LOG.rename_requests trace update_file.py databricks.yml test-schema- test-schema-renamed- plan -trace errcode $CLI bundle deploy +trace $CLI bundle deploy trace print_requests.py //unity-catalog -title "Redeploy" -trace errcode $CLI bundle plan +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/test.toml b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/test.toml index 5e20d4c81ce..e0f4bb72822 100644 --- a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/test.toml +++ b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/test.toml @@ -1,4 +1,3 @@ -Badness = "A state recorded without schema_id and volume_id (by a CLI from before they were recorded) never gets them, since a deploy with nothing to change writes no state. So the plan cannot find the schema and the volume renamed out of band, and creates them again." Cloud = true # The test rewrites the deployment state by hand, which the recording service refuses because it diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 1f099b32c2f..44d6e9f0f12 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -143,6 +143,9 @@ type PlanEntry struct { // - an UpdateWithID that only follows a parent's rename (or a rename found by the plan): // the backend already moved the resource, so applying it records the state under the // ID IDFromState gives it. It is left out of the textual plan and counted as unchanged. + // - an Update of an otherwise unchanged resource whose state lacks its internal ID (recorded + // before the ID was, or by bind): applying it records the ID from the remote state. It is + // likewise left out of the textual plan and counted as unchanged. StateOnly bool `json:"state_only,omitempty"` NewState *structvar.StructVarJSON `json:"new_state,omitempty"` RemoteState any `json:"remote_state,omitempty"` diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 30f051b043a..5b093a61d4c 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -53,6 +53,9 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, case deployplan.Recreate: return d.Recreate(ctx, db, oldID, newState) case deployplan.Update: + if planEntry.StateOnly { + return d.RecordState(ctx, db, oldID, newState, planEntry) + } return d.Update(ctx, db, oldID, newState, planEntry) case deployplan.UpdateWithID: if planEntry.StateOnly { @@ -280,6 +283,16 @@ func (d *DeploymentUnit) Rekey(ctx context.Context, db *dstate.DeploymentState, return nil } +// RecordState saves the state at its ID without a backend call. A state-only update does this to +// record the internal ID the state lacks, from the plan's remote state. +func (d *DeploymentUnit) RecordState(ctx context.Context, db *dstate.DeploymentState, id string, newState any, planEntry *deployplan.PlanEntry) error { + err := d.saveState(ctx, db, id, newState, d.DependsOn, planEntry.RemoteState) + if err != nil { + return fmt.Errorf("saving state id=%s: %w", id, err) + } + return nil +} + func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, oldID string) error { oldState, err := d.loadPersistedState(db) if err != nil { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 44e4109d362..b806c5a05d3 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -405,6 +405,19 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks // at its new ID: applying only records the state under the ID IDFromState gives it. entry.StateOnly = true } + if action == deployplan.Skip { + missing, err := missingInternalID(adapter, savedState, remoteStateComparable) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } + if missing { + // The state was recorded without the internal ID (by a CLI from before it was, or by + // bind): applying only records it, from the remote state. + action = deployplan.Update + entry.StateOnly = true + } + } // Note, this unconditionally stores remoteState. However, it may updated post-deploy, so whether // it can be used for variable resolution depends on several factors, see canReadRemoteCache in LookupReferencePreDeploy @@ -881,6 +894,20 @@ func findMoved(ctx context.Context, adapter *dresources.Adapter, sv *structvar.S return id, remote, nil } +// missingInternalID reports whether savedState lacks the internal ID that remote, the remote state +// remapped to the state type, has. +func missingInternalID(adapter *dresources.Adapter, savedState, remote any) (bool, error) { + if !adapter.HasInternalID() || remote == nil { + return false, nil + } + saved, err := adapter.InternalID(savedState) + if err != nil || saved != "" { + return false, err + } + found, err := adapter.InternalID(remote) + return found != "", err +} + // hasUnresolvedIDField reports whether a field the ID is built from is still an unresolved reference. func hasUnresolvedIDField(adapter *dresources.Adapter, sv *structvar.StructVar) bool { for pathString := range sv.Refs { From 2a49a4c625f27fb6630ff66a87dc1e911f45e0bc Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 11 Nov 2026 09:21:53 +0100 Subject: [PATCH 52/75] statemgmt: record internal IDs when migrating from terraform on deploy (review A3) The terraform state has no volume_id, and the migrated state would otherwise get the IDs from a state-only update in the next plan. The migration's plan check already read every resource, so record them from there. Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 37 ++++++++++++++++++++++++++++ bundle/statemgmt/direct_migration.go | 4 +++ 2 files changed, 41 insertions(+) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index b806c5a05d3..88a7e11072f 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -908,6 +908,43 @@ func missingInternalID(adapter *dresources.Adapter, savedState, remote any) (boo return found != "", err } +// RecordMissingInternalIDs adds to data the internal IDs that plan's state-only updates would +// record (see missingInternalID). A migration uses it so the state it converts has them from the +// start, instead of a state-only update in the next plan. +func RecordMissingInternalIDs(plan *deployplan.Plan, data *dstate.Database) error { + adapters, err := dresources.InitAll(nil) + if err != nil { + return err + } + b := DeploymentBundle{Adapters: adapters} + for resourceKey, entry := range plan.Plan { + if entry.Action != deployplan.Update || !entry.StateOnly { + continue + } + record, ok := data.State[resourceKey] + if !ok { + continue + } + adapter, err := b.getAdapterForKey(resourceKey) + if err != nil { + return err + } + state, err := parseState(adapter.StateType(), record.State) + if err != nil { + return fmt.Errorf("%s: interpreting state: %w", resourceKey, err) + } + if err := adapter.RecordInternalID(state, entry.RemoteState); err != nil { + return fmt.Errorf("%s: recording internal id: %w", resourceKey, err) + } + record.State, err = json.Marshal(state) + if err != nil { + return fmt.Errorf("%s: %w", resourceKey, err) + } + data.State[resourceKey] = record + } + return nil +} + // hasUnresolvedIDField reports whether a field the ID is built from is still an unresolved reference. func hasUnresolvedIDField(adapter *dresources.Adapter, sv *structvar.StructVar) bool { for pathString := range sv.Refs { diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index 4fefd8504ac..64c78cf10df 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -108,6 +108,10 @@ func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) { if err := json.Unmarshal(raw, &data); err != nil { return false, fmt.Errorf("parsing migrated state: %w", err) } + // The terraform state has no internal IDs (schema_id, volume_id); the plan check read them. + if err := direct.RecordMissingInternalIDs(plan, &data); err != nil { + return false, fmt.Errorf("recording internal ids in migrated state: %w", err) + } b.DeploymentBundle.StateDB.OpenWithData(localDirectPath, data) // Announce the migration only once the conversion and plan check succeeded, so a run with no From e5222b31289ac92a2edd241a38576d62d96385dc Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 11 Nov 2026 13:26:36 +0100 Subject: [PATCH 53/75] migrate: take schema_id from the terraform state (review A3) The provider records UC's schema_id on databricks_schema. databricks_volume has no volume_id, so a volume migrated this way has none. Co-authored-by: Isaac --- bundle/migrate/build_state.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index be8132fd82d..7c625acf1d9 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -257,6 +257,22 @@ func BuildStateFromTF( } } + // The internal ID, where the terraform provider records it under the same name (schema_id; + // databricks_volume has no volume_id, so a migrated volume has none). + if adapter.HasInternalID() { + path, err := structpath.ParsePath(adapter.ResourceConfig().InternalIDField) + if err != nil { + return warningsSeen, fmt.Errorf("%s: %w", node, err) + } + if v, err := LookupTFField(tfAttrs, group, srcName, path); err == nil { + if internalID, ok := v.(string); ok && internalID != "" { + if err := structaccess.Set(sv.Value, path, internalID); err != nil { + return warningsSeen, fmt.Errorf("%s: cannot set %s: %w", node, path, err) + } + } + } + } + // Reconcile id-composing fields with the deployed terraform state so a pending id // change surfaces in the plan instead of being snapshotted as already applied. A no-op // for permissions/grants sub-nodes: their adapters declare no id fields. From 3ce8cf4466251419f517f634e3991f74bb7cfc4f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 11 Nov 2026 13:26:38 +0100 Subject: [PATCH 54/75] acc: the invariant replay tests do not repeat the synced table init's requests The fixture was captured with the init script, so replaying it after the init created the source schema twice, which the test server now rejects as UC does. Co-authored-by: Isaac --- acceptance/bundle/invariant/auto-migrate/script | 2 ++ .../invariant/configs/synced_database_table.yml.tmpl-init.sh | 5 +++++ acceptance/bundle/invariant/migrate/script | 2 ++ 3 files changed, 9 insertions(+) diff --git a/acceptance/bundle/invariant/auto-migrate/script b/acceptance/bundle/invariant/auto-migrate/script index 9ca136c4881..0ed29174023 100644 --- a/acceptance/bundle/invariant/auto-migrate/script +++ b/acceptance/bundle/invariant/auto-migrate/script @@ -3,6 +3,8 @@ unset DATABRICKS_BUNDLE_ENGINE +# The fixture replays the init script's requests too (see synced_database_table's init). +export INVARIANT_REPLAY=1 invariant_setup # Replay a prior terraform deployment (per-config fixture captured once from a real terraform diff --git a/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl-init.sh b/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl-init.sh index 4a47f8aa2bd..022c7ac3700 100755 --- a/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl-init.sh +++ b/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl-init.sh @@ -1,5 +1,10 @@ #!/bin/bash +# A replayed terraform fixture already makes these requests: it was captured with this script. +if [ -n "$INVARIANT_REPLAY" ]; then + return +fi + # Create a unique source table for this test run to avoid hitting the 20-table-per-source limit echo "Creating temporary source table: main.test_synced_$UNIQUE_NAME.trips_source" diff --git a/acceptance/bundle/invariant/migrate/script b/acceptance/bundle/invariant/migrate/script index a81c1bdfe84..52571839cf6 100644 --- a/acceptance/bundle/invariant/migrate/script +++ b/acceptance/bundle/invariant/migrate/script @@ -3,6 +3,8 @@ unset DATABRICKS_BUNDLE_ENGINE +# The fixture replays the init script's requests too (see synced_database_table's init). +export INVARIANT_REPLAY=1 invariant_setup # Replay a prior terraform deployment (per-config fixture captured once from a real terraform run), From aedd0ad73a16ca7a755baabd53d1d0d122b1f6b1 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 11 Nov 2026 16:41:41 +0100 Subject: [PATCH 55/75] acc: the synced table terraform fixture holds only terraform's requests It was captured with the init script's schema and source-table requests, which every test runs anyway, so the replay created the schema twice. Reverts the INVARIANT_REPLAY workaround. Co-authored-by: Isaac --- acceptance/bundle/invariant/auto-migrate/script | 2 -- .../synced_database_table.yml.tmpl-init.sh | 5 ----- ...synced_database_table.yml.tmpl.requests.json | 17 ----------------- acceptance/bundle/invariant/migrate/script | 2 -- 4 files changed, 26 deletions(-) diff --git a/acceptance/bundle/invariant/auto-migrate/script b/acceptance/bundle/invariant/auto-migrate/script index 0ed29174023..9ca136c4881 100644 --- a/acceptance/bundle/invariant/auto-migrate/script +++ b/acceptance/bundle/invariant/auto-migrate/script @@ -3,8 +3,6 @@ unset DATABRICKS_BUNDLE_ENGINE -# The fixture replays the init script's requests too (see synced_database_table's init). -export INVARIANT_REPLAY=1 invariant_setup # Replay a prior terraform deployment (per-config fixture captured once from a real terraform diff --git a/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl-init.sh b/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl-init.sh index 022c7ac3700..4a47f8aa2bd 100755 --- a/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl-init.sh +++ b/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl-init.sh @@ -1,10 +1,5 @@ #!/bin/bash -# A replayed terraform fixture already makes these requests: it was captured with this script. -if [ -n "$INVARIANT_REPLAY" ]; then - return -fi - # Create a unique source table for this test run to avoid hitting the 20-table-per-source limit echo "Creating temporary source table: main.test_synced_$UNIQUE_NAME.trips_source" diff --git a/acceptance/bundle/invariant/fixtures/synced_database_table.yml.tmpl.requests.json b/acceptance/bundle/invariant/fixtures/synced_database_table.yml.tmpl.requests.json index 9807dd442cb..0d373f470c9 100644 --- a/acceptance/bundle/invariant/fixtures/synced_database_table.yml.tmpl.requests.json +++ b/acceptance/bundle/invariant/fixtures/synced_database_table.yml.tmpl.requests.json @@ -1,21 +1,4 @@ [ - { - "method": "POST", - "path": "/api/2.1/unity-catalog/schemas", - "body": { - "catalog_name": "main", - "name": "test_synced_[UNIQUE_NAME]" - } - }, - { - "method": "POST", - "path": "/api/2.0/sql/statements/", - "body": { - "statement": "CREATE TABLE main.test_synced_[UNIQUE_NAME].trips_source AS SELECT * FROM samples.nyctaxi.trips LIMIT 10", - "wait_timeout": "45s", - "warehouse_id": "8ec9edc1-db0c-40df-af8d-7580020fe61e" - } - }, { "method": "POST", "path": "/api/2.0/workspace/mkdirs", diff --git a/acceptance/bundle/invariant/migrate/script b/acceptance/bundle/invariant/migrate/script index 52571839cf6..a81c1bdfe84 100644 --- a/acceptance/bundle/invariant/migrate/script +++ b/acceptance/bundle/invariant/migrate/script @@ -3,8 +3,6 @@ unset DATABRICKS_BUNDLE_ENGINE -# The fixture replays the init script's requests too (see synced_database_table's init). -export INVARIANT_REPLAY=1 invariant_setup # Replay a prior terraform deployment (per-config fixture captured once from a real terraform run), From eaf39083389b2e4f5b0604cd7af81a7a0272e83e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 11 Nov 2026 22:30:20 +0100 Subject: [PATCH 56/75] migrate: read volume_id for bundle deployment migrate (review A3) The command plans nothing and the terraform state has no volume_id, so it reads each resource whose migrated state lacks its internal ID. A failed read only warns: the next deploy records the ID with a state-only update. Co-authored-by: Isaac --- .../migrate/command/basic/out.new_state.json | 1 + .../command/basic/out.plan_update.json | 1 + .../migrate/command/grants/out.new_state.json | 6 +- bundle/direct/bundle_plan.go | 65 +++++++++++++++++-- cmd/bundle/deployment/migrate.go | 26 ++++++++ 5 files changed, 90 insertions(+), 9 deletions(-) diff --git a/acceptance/bundle/migrate/command/basic/out.new_state.json b/acceptance/bundle/migrate/command/basic/out.new_state.json index e3945a2c940..30204653983 100644 --- a/acceptance/bundle/migrate/command/basic/out.new_state.json +++ b/acceptance/bundle/migrate/command/basic/out.new_state.json @@ -87,6 +87,7 @@ "catalog_name": "mycat", "name": "myvol", "schema_name": "myschema", + "volume_id": "[UUID][3]", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/migrate/command/basic/out.plan_update.json b/acceptance/bundle/migrate/command/basic/out.plan_update.json index 4d7b49372d5..f7d9eb2bf15 100644 --- a/acceptance/bundle/migrate/command/basic/out.plan_update.json +++ b/acceptance/bundle/migrate/command/basic/out.plan_update.json @@ -244,6 +244,7 @@ "volume_id": { "action": "skip", "reason": "internal_id", + "old": "[UUID][2]", "remote": "[UUID][2]" } } diff --git a/acceptance/bundle/migrate/command/grants/out.new_state.json b/acceptance/bundle/migrate/command/grants/out.new_state.json index 82b918f76d3..eb3b28ae467 100644 --- a/acceptance/bundle/migrate/command/grants/out.new_state.json +++ b/acceptance/bundle/migrate/command/grants/out.new_state.json @@ -1,7 +1,7 @@ { "state_version": 3, "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][0]", "serial": 9, "state": { "resources.registered_models.my_registered_model": { @@ -44,7 +44,8 @@ "__id__": "main.schema_grants", "state": { "catalog_name": "main", - "name": "schema_grants" + "name": "schema_grants", + "schema_id": "[UUID][1]" } }, "resources.schemas.my_schema.grants": { @@ -75,6 +76,7 @@ "catalog_name": "main", "name": "volume_name", "schema_name": "schema_grants", + "volume_id": "[UUID][2]", "volume_type": "MANAGED" }, "depends_on": [ diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 88a7e11072f..dd402f0add8 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -921,30 +921,81 @@ func RecordMissingInternalIDs(plan *deployplan.Plan, data *dstate.Database) erro if entry.Action != deployplan.Update || !entry.StateOnly { continue } - record, ok := data.State[resourceKey] - if !ok { + if _, ok := data.State[resourceKey]; !ok { continue } adapter, err := b.getAdapterForKey(resourceKey) if err != nil { return err } + if err := recordInternalID(adapter, data, resourceKey, entry.RemoteState); err != nil { + return err + } + } + return nil +} + +// ReadMissingInternalIDs reads each resource whose state in data lacks its internal ID, and records +// it. bundle deployment migrate uses it: it plans nothing, and the terraform state has no volume_id. +func ReadMissingInternalIDs(ctx context.Context, client *databricks.WorkspaceClient, data *dstate.Database) error { + adapters, err := dresources.InitAll(client) + if err != nil { + return err + } + b := DeploymentBundle{Adapters: adapters} + for _, resourceKey := range slices.Sorted(maps.Keys(data.State)) { + adapter, err := b.getAdapterForKey(resourceKey) + if err != nil { + return err + } + if !adapter.HasInternalID() { + continue + } + record := data.State[resourceKey] state, err := parseState(adapter.StateType(), record.State) if err != nil { return fmt.Errorf("%s: interpreting state: %w", resourceKey, err) } - if err := adapter.RecordInternalID(state, entry.RemoteState); err != nil { - return fmt.Errorf("%s: recording internal id: %w", resourceKey, err) + if internalID, err := adapter.InternalID(state); err != nil || internalID != "" { + continue + } + remote, err := retry.With(ctx, retry.IsTransientRead, func() (any, error) { + return adapter.DoRead(ctx, record.ID) + }) + if apierr.IsMissing(err) { + // Gone: the next plan creates or drops it. + continue } - record.State, err = json.Marshal(state) if err != nil { - return fmt.Errorf("%s: %w", resourceKey, err) + // The ID is optional: without it a state-only update in the next deploy records it. + log.Warnf(ctx, "%s: reading id=%q to record its internal id: %s", resourceKey, record.ID, err) + continue + } + if err := recordInternalID(adapter, data, resourceKey, remote); err != nil { + return err } - data.State[resourceKey] = record } return nil } +// recordInternalID records in the state of resourceKey in data the internal ID of remote. +func recordInternalID(adapter *dresources.Adapter, data *dstate.Database, resourceKey string, remote any) error { + record := data.State[resourceKey] + state, err := parseState(adapter.StateType(), record.State) + if err != nil { + return fmt.Errorf("%s: interpreting state: %w", resourceKey, err) + } + if err := adapter.RecordInternalID(state, remote); err != nil { + return fmt.Errorf("%s: recording internal id: %w", resourceKey, err) + } + record.State, err = json.Marshal(state) + if err != nil { + return fmt.Errorf("%s: %w", resourceKey, err) + } + data.State[resourceKey] = record + return nil +} + // hasUnresolvedIDField reports whether a field the ID is built from is still an unresolved reference. func hasUnresolvedIDField(adapter *dresources.Adapter, sv *structvar.StructVar) bool { for pathString := range sv.Refs { diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index f329ec7962c..548c54a303a 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -1,6 +1,8 @@ package deployment import ( + "context" + "encoding/json" "errors" "fmt" "os" @@ -9,6 +11,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" + "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/migrate" @@ -169,6 +172,10 @@ New bundles use the direct engine automatically, no migration is needed.`) return root.ErrAlreadyPrinted } + if err := readMissingInternalIDs(ctx, b, tempStatePath); err != nil { + return fmt.Errorf("recording internal ids: %w", err) + } + if err := os.Rename(tempStatePath, localPath); err != nil { return fmt.Errorf("renaming %s to %s: %w", tempStatePath, localPath, err) } @@ -195,3 +202,22 @@ To undo the migration, remove %s and rename %s to %s return cmd } + +// readMissingInternalIDs reads the resources whose migrated state at path lacks their internal ID, +// and records it there: the terraform state has no volume_id. +func readMissingInternalIDs(ctx context.Context, b *bundle.Bundle, path string) error { + raw, err := os.ReadFile(path) + if err != nil { + return err + } + var data dstate.Database + if err := json.Unmarshal(raw, &data); err != nil { + return fmt.Errorf("parsing %s: %w", path, err) + } + if err := direct.ReadMissingInternalIDs(ctx, b.WorkspaceClient(ctx), &data); err != nil { + return err + } + var db dstate.DeploymentState + db.OpenWithData(path, data) + return db.Persist() +} From 2bf2ffdf7a08808774379e6336ad1235ec6f6ab5 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 11 Nov 2026 22:32:17 +0100 Subject: [PATCH 57/75] acc: deploy the old-state and recreate-in-renamed-schema tests from a saved plan too (READPLAN) Co-authored-by: Isaac --- .../schemas/rename_out_of_band_old_state/out.test.toml | 1 + .../resources/schemas/rename_out_of_band_old_state/script | 6 +++--- .../schemas/rename_out_of_band_old_state/test.toml | 7 +++++++ .../volumes/recreate_in_renamed_schema/out.test.toml | 1 + .../resources/volumes/recreate_in_renamed_schema/script | 4 ++-- .../resources/volumes/recreate_in_renamed_schema/test.toml | 7 +++++++ 6 files changed, 21 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/out.test.toml b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/out.test.toml index f68b4a89c36..de136de256d 100644 --- a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/out.test.toml +++ b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DMS = [""] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/script b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/script index bd43dd908ba..3fac4e01940 100644 --- a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/script +++ b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/script @@ -16,7 +16,7 @@ trap cleanup EXIT # Drop the remote_state fields that differ between clouds or from the test server. plan() { - trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location + trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location } recorded_ids() { @@ -36,7 +36,7 @@ recorded_ids title "A deploy with nothing else to change records them, with a state-only update" plan -trace $CLI bundle deploy +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) trace print_requests.py //unity-catalog recorded_ids @@ -45,7 +45,7 @@ trace $CLI schemas update "$OLD_SCHEMA" --new-name "test-schema-renamed-${UNIQUE print_requests.py //unity-catalog &> LOG.rename_requests trace update_file.py databricks.yml test-schema- test-schema-renamed- plan -trace $CLI bundle deploy +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) trace print_requests.py //unity-catalog title "Redeploy is a no-op" diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/test.toml b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/test.toml index e0f4bb72822..b5c8f86c637 100644 --- a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/test.toml +++ b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/test.toml @@ -1,4 +1,5 @@ Cloud = true +EnvMatrix.READPLAN = ["", "1"] # The test rewrites the deployment state by hand, which the recording service refuses because it # no longer matches the deployment it has recorded, so run without deployment history. @@ -9,3 +10,9 @@ Ignore = [ ".gitignore", "databricks.yml", ] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' diff --git a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/out.test.toml b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/out.test.toml index a927a5fbc06..f7fafbcc731 100644 --- a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/out.test.toml +++ b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/script b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/script index 57d49e071b5..62efd207163 100644 --- a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/script +++ b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/script @@ -15,8 +15,8 @@ print_requests.py //unity-catalog &> LOG.deploy_requests title "Rename the schema and change the volume's type in one deploy. The type change recreates the volume" trace update_file.py databricks.yml test-schema- test-schema-renamed- trace update_file.py databricks.yml "volume_type: MANAGED" "volume_type: EXTERNAL" -trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at -trace $CLI bundle deploy --auto-approve +trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py created_at updated_at +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) --auto-approve trace print_requests.py //unity-catalog title "Redeploy is a no-op" diff --git a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/test.toml b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/test.toml index 263d5495591..cc6e6bc9c1f 100644 --- a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/test.toml +++ b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/test.toml @@ -1,3 +1,4 @@ +EnvMatrix.READPLAN = ["", "1"] # Local only: an EXTERNAL volume needs an external location, which the test workspaces do not have. # volumes/move_from_renamed_schema covers the same recreate on a real workspace. @@ -7,3 +8,9 @@ Ignore = [ ".gitignore", "databricks.yml", ] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' From cb5c958dcaaec2a74aef9364320e21c173197ca3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 11 Nov 2026 22:35:24 +0100 Subject: [PATCH 58/75] acc: destroy after an out-of-band rename, and a foreign schema at the recorded name (review B1, B2, Badness) Both gaps are on main too: destroy never looks for a moved resource, and the plan takes whatever is at the recorded ID without checking schema_id. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 13 +++ .../out.test.toml | 2 + .../output.txt | 32 +++++++ .../destroy_after_rename_out_of_band/script | 30 +++++++ .../test.toml | 8 ++ .../databricks.yml.tmpl | 9 ++ .../foreign_at_recorded_name/out.test.toml | 2 + .../foreign_at_recorded_name/output.txt | 84 +++++++++++++++++++ .../schemas/foreign_at_recorded_name/script | 40 +++++++++ .../foreign_at_recorded_name/test.toml | 8 ++ 10 files changed, 228 insertions(+) create mode 100644 acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/output.txt create mode 100644 acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/script create mode 100644 acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/test.toml create mode 100644 acceptance/bundle/resources/schemas/foreign_at_recorded_name/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/foreign_at_recorded_name/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/foreign_at_recorded_name/output.txt create mode 100644 acceptance/bundle/resources/schemas/foreign_at_recorded_name/script create mode 100644 acceptance/bundle/resources/schemas/foreign_at_recorded_name/test.toml diff --git a/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..2234fdf2e39 --- /dev/null +++ b/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/out.test.toml b/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/output.txt b/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/output.txt new file mode 100644 index 00000000000..b6c56a0ee49 --- /dev/null +++ b/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/output.txt @@ -0,0 +1,32 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema out of band, and the config to match +>>> [CLI] schemas update main.test-schema-[UNIQUE_NAME] --new-name test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" +} + +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +=== Destroy +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 0 deleted + +>>> print_requests.py //unity-catalog + +=== The renamed schema is still there +>>> [CLI] schemas get main.test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" +} diff --git a/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/script b/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/script new file mode 100644 index 00000000000..a0c969ce99c --- /dev/null +++ b/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/script @@ -0,0 +1,30 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_SCHEMA="main.test-schema-${UNIQUE_NAME}" +NEW_SCHEMA="main.test-schema-renamed-${UNIQUE_NAME}" + +cleanup() { + # Whatever destroy left behind. + errcode $CLI schemas delete "$NEW_SCHEMA" --force > LOG.cleanup 2>&1 + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "$OLD_SCHEMA" | jq -r .schema_id)" SCHEMA_ID + +title "Rename the schema out of band, and the config to match" +trace $CLI schemas update "$OLD_SCHEMA" --new-name "test-schema-renamed-${UNIQUE_NAME}" | jq '{full_name, schema_id}' +print_requests.py //unity-catalog &> LOG.rename_requests +trace update_file.py databricks.yml test-schema- test-schema-renamed- + +title "Destroy" +trace $CLI bundle destroy --auto-approve +trace print_requests.py //unity-catalog + +title "The renamed schema is still there" +trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, schema_id}' diff --git a/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/test.toml b/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/test.toml new file mode 100644 index 00000000000..a056278311d --- /dev/null +++ b/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/test.toml @@ -0,0 +1,8 @@ +Badness = "Destroy does not look for a resource renamed outside the bundle: it reads the recorded name, finds nothing and drops the state entry, so the renamed schema and its volume are left behind. Same on main." +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/foreign_at_recorded_name/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/foreign_at_recorded_name/databricks.yml.tmpl new file mode 100644 index 00000000000..d73aacf0d14 --- /dev/null +++ b/acceptance/bundle/resources/schemas/foreign_at_recorded_name/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + comment: c1 diff --git a/acceptance/bundle/resources/schemas/foreign_at_recorded_name/out.test.toml b/acceptance/bundle/resources/schemas/foreign_at_recorded_name/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/foreign_at_recorded_name/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/foreign_at_recorded_name/output.txt b/acceptance/bundle/resources/schemas/foreign_at_recorded_name/output.txt new file mode 100644 index 00000000000..abe10612e29 --- /dev/null +++ b/acceptance/bundle/resources/schemas/foreign_at_recorded_name/output.txt @@ -0,0 +1,84 @@ + +=== Deploy a schema +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Out of band, rename the schema away, and create another one under its name +>>> [CLI] schemas update main.test-schema-[UNIQUE_NAME] --new-name test-schema-moved-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-moved-[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" +} + +=== The plan takes the other schema for the bundle's, though its schema_id differs, and plans to update it +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "update", + "new_state": { + "value": { + "catalog_name": "main", + "comment": "c1", + "name": "test-schema-[UNIQUE_NAME]" + } + }, + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-[UNIQUE_NAME]", + "name": "test-schema-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[FOREIGN_SCHEMA_ID]" + }, + "changes": { + "comment": { + "action": "update", + "old": "c1", + "new": "c1" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]", + "remote": "[FOREIGN_SCHEMA_ID]" + } + } + } +} + +=== Destroy deletes the other schema +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> print_requests.py //unity-catalog +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "q": { + "force": "true" + } +} + +>>> musterr [CLI] schemas get main.test-schema-[UNIQUE_NAME] +Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. + +=== The bundle's own schema is left behind +>>> [CLI] schemas get main.test-schema-moved-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-moved-[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" +} diff --git a/acceptance/bundle/resources/schemas/foreign_at_recorded_name/script b/acceptance/bundle/resources/schemas/foreign_at_recorded_name/script new file mode 100644 index 00000000000..0af10008bf7 --- /dev/null +++ b/acceptance/bundle/resources/schemas/foreign_at_recorded_name/script @@ -0,0 +1,40 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +SCHEMA="main.test-schema-${UNIQUE_NAME}" +MOVED_SCHEMA="main.test-schema-moved-${UNIQUE_NAME}" + +cleanup() { + # Whatever destroy left behind. + errcode $CLI schemas delete "$MOVED_SCHEMA" --force > LOG.cleanup 2>&1 + errcode $CLI schemas delete "$SCHEMA" --force >> LOG.cleanup 2>&1 + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +} + +title "Deploy a schema" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "$SCHEMA" | jq -r .schema_id)" SCHEMA_ID + +title "Out of band, rename the schema away, and create another one under its name" +trace $CLI schemas update "$SCHEMA" --new-name "test-schema-moved-${UNIQUE_NAME}" | jq '{full_name, schema_id}' +add_repl "$($CLI schemas create "test-schema-${UNIQUE_NAME}" main | jq -r .schema_id)" FOREIGN_SCHEMA_ID +print_requests.py //unity-catalog &> LOG.oob_requests + +title "The plan takes the other schema for the bundle's, though its schema_id differs, and plans to update it" +plan + +title "Destroy deletes the other schema" +trace $CLI bundle destroy --auto-approve +trace print_requests.py //unity-catalog +trace musterr $CLI schemas get "$SCHEMA" + +title "The bundle's own schema is left behind" +trace $CLI schemas get "$MOVED_SCHEMA" | jq '{full_name, schema_id}' diff --git a/acceptance/bundle/resources/schemas/foreign_at_recorded_name/test.toml b/acceptance/bundle/resources/schemas/foreign_at_recorded_name/test.toml new file mode 100644 index 00000000000..f8c09b56f6e --- /dev/null +++ b/acceptance/bundle/resources/schemas/foreign_at_recorded_name/test.toml @@ -0,0 +1,8 @@ +Badness = "The plan reads the recorded name and takes whatever schema is there, without checking schema_id: one created there by someone else after ours was renamed is updated and then deleted by destroy, and ours is left behind. Same on main." +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] From 6041de65c4dc6dca740ed0adf8ca6f92a519d777 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 11 Nov 2026 22:36:51 +0100 Subject: [PATCH 59/75] acc: a forbidden read at the config's name fails the plan (review B3, Badness) Co-authored-by: Isaac --- .../rename_probe_forbidden/databricks.yml | 8 +++++ .../rename_probe_forbidden/out.test.toml | 2 ++ .../schemas/rename_probe_forbidden/output.txt | 30 +++++++++++++++++++ .../schemas/rename_probe_forbidden/script | 12 ++++++++ .../schemas/rename_probe_forbidden/test.toml | 13 ++++++++ 5 files changed, 65 insertions(+) create mode 100644 acceptance/bundle/resources/schemas/rename_probe_forbidden/databricks.yml create mode 100644 acceptance/bundle/resources/schemas/rename_probe_forbidden/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_probe_forbidden/output.txt create mode 100644 acceptance/bundle/resources/schemas/rename_probe_forbidden/script create mode 100644 acceptance/bundle/resources/schemas/rename_probe_forbidden/test.toml diff --git a/acceptance/bundle/resources/schemas/rename_probe_forbidden/databricks.yml b/acceptance/bundle/resources/schemas/rename_probe_forbidden/databricks.yml new file mode 100644 index 00000000000..25d1c7a57cb --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_probe_forbidden/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: test-bundle + +resources: + schemas: + schema1: + catalog_name: main + name: myschema diff --git a/acceptance/bundle/resources/schemas/rename_probe_forbidden/out.test.toml b/acceptance/bundle/resources/schemas/rename_probe_forbidden/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_probe_forbidden/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_probe_forbidden/output.txt b/acceptance/bundle/resources/schemas/rename_probe_forbidden/output.txt new file mode 100644 index 00000000000..1bf3b5d0481 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_probe_forbidden/output.txt @@ -0,0 +1,30 @@ + +=== Deploy a schema +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created schemas.schema1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Delete the schema out of band and rename it in the config. Reading the new name is forbidden +>>> [CLI] schemas delete main.myschema + +>>> update_file.py databricks.yml name: myschema name: myschema2 + +>>> errcode [CLI] bundle plan +Error: cannot plan resources.schemas.schema1: reading id="main.myschema2": User does not have USE SCHEMA on Schema main.myschema2 (403 PERMISSION_DENIED) + +Endpoint: GET [DATABRICKS_URL]/api/2.1/unity-catalog/schemas/main.myschema2? +HTTP Status: 403 Forbidden +API error_code: PERMISSION_DENIED +API message: User does not have USE SCHEMA on Schema main.myschema2 + +Error: planning failed + + +Exit code: 1 + +>>> errcode [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 0 deleted diff --git a/acceptance/bundle/resources/schemas/rename_probe_forbidden/script b/acceptance/bundle/resources/schemas/rename_probe_forbidden/script new file mode 100644 index 00000000000..02af2923eed --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_probe_forbidden/script @@ -0,0 +1,12 @@ +cleanup() { + trace errcode $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy a schema" +trace $CLI bundle deploy + +title "Delete the schema out of band and rename it in the config. Reading the new name is forbidden" +trace $CLI schemas delete main.myschema +trace update_file.py databricks.yml "name: myschema" "name: myschema2" +trace errcode $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_probe_forbidden/test.toml b/acceptance/bundle/resources/schemas/rename_probe_forbidden/test.toml new file mode 100644 index 00000000000..68fab62f0cf --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_probe_forbidden/test.toml @@ -0,0 +1,13 @@ +Badness = "When the schema is gone from its recorded name, the plan looks for it under the name in the config, and an error other than not-found there fails the plan. Before that lookup existed the plan created the schema." + +# Local only: the 403 on the new name is injected. + +Ignore = [ + ".databricks", + ".gitignore", +] + +[[Server]] +Pattern = "GET /api/2.1/unity-catalog/schemas/main.myschema2" +Response.StatusCode = 403 +Response.Body = '{"error_code": "PERMISSION_DENIED", "message": "User does not have USE SCHEMA on Schema main.myschema2"}' From b87ef48637acf1a6664fe4dad7a5e3881881b8f2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 11 Nov 2026 22:38:13 +0100 Subject: [PATCH 60/75] direct: a failed lookup for a moved resource only warns (review B3) Before the lookup existed the plan created the resource; an error other than not-found at the config's ID now leads to the same plan, with a warning. Co-authored-by: Isaac --- .../schemas/rename_probe_forbidden/output.txt | 17 +++++------------ .../schemas/rename_probe_forbidden/script | 4 ++-- .../schemas/rename_probe_forbidden/test.toml | 2 -- bundle/direct/bundle_plan.go | 5 ++++- 4 files changed, 11 insertions(+), 17 deletions(-) diff --git a/acceptance/bundle/resources/schemas/rename_probe_forbidden/output.txt b/acceptance/bundle/resources/schemas/rename_probe_forbidden/output.txt index 1bf3b5d0481..87df5227de4 100644 --- a/acceptance/bundle/resources/schemas/rename_probe_forbidden/output.txt +++ b/acceptance/bundle/resources/schemas/rename_probe_forbidden/output.txt @@ -6,23 +6,16 @@ Created schemas.schema1 Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -=== Delete the schema out of band and rename it in the config. Reading the new name is forbidden +=== Delete the schema out of band and rename it in the config. Reading the new name is forbidden, so the plan only warns and creates the schema >>> [CLI] schemas delete main.myschema >>> update_file.py databricks.yml name: myschema name: myschema2 ->>> errcode [CLI] bundle plan -Error: cannot plan resources.schemas.schema1: reading id="main.myschema2": User does not have USE SCHEMA on Schema main.myschema2 (403 PERMISSION_DENIED) +>>> [CLI] bundle plan +Warn: planning resources.schemas.schema1: looking for a moved resource at id="main.myschema2": User does not have USE SCHEMA on Schema main.myschema2 +create schemas.schema1 -Endpoint: GET [DATABRICKS_URL]/api/2.1/unity-catalog/schemas/main.myschema2? -HTTP Status: 403 Forbidden -API error_code: PERMISSION_DENIED -API message: User does not have USE SCHEMA on Schema main.myschema2 - -Error: planning failed - - -Exit code: 1 +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged >>> errcode [CLI] bundle destroy --auto-approve All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/schemas/rename_probe_forbidden/script b/acceptance/bundle/resources/schemas/rename_probe_forbidden/script index 02af2923eed..393aae46e8b 100644 --- a/acceptance/bundle/resources/schemas/rename_probe_forbidden/script +++ b/acceptance/bundle/resources/schemas/rename_probe_forbidden/script @@ -6,7 +6,7 @@ trap cleanup EXIT title "Deploy a schema" trace $CLI bundle deploy -title "Delete the schema out of band and rename it in the config. Reading the new name is forbidden" +title "Delete the schema out of band and rename it in the config. Reading the new name is forbidden, so the plan only warns and creates the schema" trace $CLI schemas delete main.myschema trace update_file.py databricks.yml "name: myschema" "name: myschema2" -trace errcode $CLI bundle plan +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_probe_forbidden/test.toml b/acceptance/bundle/resources/schemas/rename_probe_forbidden/test.toml index 68fab62f0cf..29e0279e5f6 100644 --- a/acceptance/bundle/resources/schemas/rename_probe_forbidden/test.toml +++ b/acceptance/bundle/resources/schemas/rename_probe_forbidden/test.toml @@ -1,5 +1,3 @@ -Badness = "When the schema is gone from its recorded name, the plan looks for it under the name in the config, and an error other than not-found there fails the plan. Before that lookup existed the plan created the schema." - # Local only: the 403 on the new name is injected. Ignore = [ diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index dd402f0add8..2817f29f44e 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -881,7 +881,10 @@ func findMoved(ctx context.Context, adapter *dresources.Adapter, sv *structvar.S return "", nil, nil } if err != nil { - return "", nil, fmt.Errorf("reading id=%q: %w", id, err) + // This read only looks for the resource. Failing it fails nothing: the plan goes on as if + // it were not there, as it did before this lookup existed. + log.Warnf(ctx, "looking for a moved resource at id=%q: %s", id, err) + return "", nil, nil } found, err := adapter.RemapState(remote) if err != nil { From aa839436b51cec32b840e7de44d92a3e087ed87c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 11 Nov 2026 22:39:04 +0100 Subject: [PATCH 61/75] Mention finding a renamed schema or volume in the changelog entry Co-authored-by: Isaac --- .nextchanges/bundles/schema-rename.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/schema-rename.md b/.nextchanges/bundles/schema-rename.md index 3bc837ae5d1..f728601449b 100644 --- a/.nextchanges/bundles/schema-rename.md +++ b/.nextchanges/bundles/schema-rename.md @@ -1 +1 @@ -* Rename a schema in place when its `name` changes, instead of deleting and recreating it. Schemas and volumes inside a renamed catalog or schema now follow it to the new name. ([#7343](https://github.com/databricks/cli/pull/7343)) +* Rename a schema in place when its `name` changes, instead of deleting and recreating it. Schemas and volumes inside a renamed catalog or schema now follow it to the new name, and one renamed outside the bundle is found under its new name. ([#7343](https://github.com/databricks/cli/pull/7343)) From ccb9119dc0da2e9e7f8db1dac296f740048cc0c0 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 12 Nov 2026 00:37:00 +0100 Subject: [PATCH 62/75] acc: regenerate for title's own trailing newline (main #7003); restore schema-rename's in-place rename Co-authored-by: Isaac --- .../migrate/auto/schema-rename/output.txt | 36 ++++--------------- .../resources/catalogs/rename/output.txt | 6 ++++ .../bundle/resources/catalogs/rename/script | 2 +- .../rename_all_out_of_band/output.txt | 5 +++ .../catalogs/rename_out_of_band/output.txt | 5 +++ .../catalogs/rename_with_comment/output.txt | 3 ++ .../output.txt | 4 +++ .../foreign_at_recorded_name/output.txt | 5 +++ .../schemas/recreate_interrupted/output.txt | 5 +++ .../schemas/recreate_out_of_band/output.txt | 3 ++ .../resources/schemas/rename/output.txt | 7 ++++ .../schemas/rename_case_only/output.txt | 3 ++ .../schemas/rename_interrupted/output.txt | 5 +++ .../schemas/rename_out_of_band/output.txt | 6 ++++ .../rename_out_of_band_old_state/output.txt | 5 +++ .../schemas/rename_probe_forbidden/output.txt | 2 ++ .../schemas/rename_then_change/output.txt | 7 ++++ .../schemas/rename_with_comment/output.txt | 3 ++ .../schemas/rename_with_volume/output.txt | 7 ++++ .../schemas/rename_with_volume_ref/output.txt | 5 +++ .../move_from_renamed_schema/output.txt | 5 +++ .../recreate_in_renamed_schema/output.txt | 3 ++ .../volumes/recreate_out_of_band/output.txt | 3 ++ .../volumes/rename_case_only/output.txt | 4 +++ .../rename_catalog_and_schema/output.txt | 6 ++++ .../volumes/rename_out_of_band/output.txt | 6 ++++ 26 files changed, 120 insertions(+), 31 deletions(-) diff --git a/acceptance/bundle/migrate/auto/schema-rename/output.txt b/acceptance/bundle/migrate/auto/schema-rename/output.txt index 6acf51246ff..782a1a4c433 100644 --- a/acceptance/bundle/migrate/auto/schema-rename/output.txt +++ b/acceptance/bundle/migrate/auto/schema-rename/output.txt @@ -3,40 +3,16 @@ >>> replay_tfstate.py terraform-requests.json terraform.tfstate -=== Rename the schema (an immutable id field). Without --auto-approve the recreate is refused +=== Rename the schema (an updatable id field) and deploy on direct ->>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy -Warn: migration to direct: schemas.my_schema: name differs between config ("myschema-[UNIQUE_NAME]-renamed") and terraform state ("myschema-[UNIQUE_NAME]"); this resource will be recreated. -Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... - -This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: - recreate resources.schemas.my_schema -Warn: Migration not committed, keeping Terraform state -Error: the deployment requires destructive actions, but the current console does not support prompting. -Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. -To proceed, use --auto-approve after reviewing the plan above. - -Files: 2 uploaded, 0 deleted - -Exit code: 1 - ->>> print_migration_telemetry -direct_migrate_recreate_planned true - -=== With --auto-approve the recreate is applied on direct under the new name - ->>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy --auto-approve -Warn: migration to direct: schemas.my_schema: name differs between config ("myschema-[UNIQUE_NAME]-renamed") and terraform state ("myschema-[UNIQUE_NAME]"); this resource will be recreated. +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Warn: migration to direct: schemas.my_schema: name differs between config ("myschema-[UNIQUE_NAME]-renamed") and terraform state ("myschema-[UNIQUE_NAME]"); this resource will be renamed. Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... - -This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: - recreate resources.schemas.my_schema Migrated 1 resource to direct deployment engine. -Recreated schemas.my_schema -Files: 0 uploaded, 0 deleted -Resources: 1 created, 0 changed, 1 deleted, 0 unchanged +Updated schemas.my_schema +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Redeploy on direct: the schema now matches config, so the plan is a clean no-op diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index e07288217d4..0d5b9fcb881 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -1,5 +1,6 @@ === Deploy a catalog with a schema and a volume in it + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created catalogs.catalog1 @@ -36,9 +37,11 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } === Rename the catalog + >>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ === UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them + >>> [CLI] bundle plan --output json { "resources.catalogs.catalog1": { @@ -158,6 +161,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } === Deploy renames the catalog. UC moves the schema and the volume with it, so they get no request + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated catalogs.catalog1 @@ -193,6 +197,7 @@ old catalog is gone } === The state tracks the new paths + >>> [CLI] bundle summary --output json { "catalogs": { @@ -207,6 +212,7 @@ old catalog is gone } === Redeploy is a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script index fea2b758bf0..1b20c07091b 100644 --- a/acceptance/bundle/resources/catalogs/rename/script +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -28,7 +28,7 @@ title "Deploy renames the catalog. UC moves the schema and the volume with it, s trace $CLI bundle deploy $(readplanarg .databricks/plan.json) trace print_requests.py //unity-catalog -title "The old catalog is gone and the schema and the volume are under the new one, with their original ids\n" +title "The old catalog is gone and the schema and the volume are under the new one, with their original ids" # Not traced: the "not found" message differs between UC and the test server. $CLI catalogs get "$OLD_CATALOG" &> /dev/null && echo "old catalog still exists" || echo "old catalog is gone" trace $CLI schemas get "$NEW_CATALOG.myschema" | jq '{full_name, catalog_name, schema_id}' diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt index feb14bcca39..af5dcff5840 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt @@ -1,5 +1,6 @@ === Deploy a catalog with a schema and a volume in it + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created catalogs.catalog1 @@ -9,6 +10,7 @@ Files: 2 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Rename the volume, the schema and the catalog out of band + >>> [CLI] volumes update test_catalog_[UNIQUE_NAME].myschema.myvolume --new-name myvolume2 { "full_name": "test_catalog_[UNIQUE_NAME].myschema.myvolume2" @@ -25,6 +27,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } === Rename all three in the config to match + >>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ >>> update_file.py databricks.yml name: myschema name: myschema2 @@ -32,6 +35,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml name: myvolume name: myvolume2 === The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again + >>> [CLI] bundle plan --output json { "resources.catalogs.catalog1": { @@ -165,6 +169,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } === The catalog's create fails because the renamed catalog has its name + >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Error: cannot create resources.catalogs.catalog1: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists (400 CATALOG_ALREADY_EXISTS) diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt index 529021aa62d..cf94276e0cd 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt @@ -1,5 +1,6 @@ === Deploy a catalog with a schema and a volume in it + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created catalogs.catalog1 @@ -9,15 +10,18 @@ Files: 2 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Rename the catalog out of band. UC moves the schema and the volume with it + >>> [CLI] catalogs update test_catalog_[UNIQUE_NAME] --new-name test_catalog_renamed_[UNIQUE_NAME] { "name": "test_catalog_renamed_[UNIQUE_NAME]" } === Rename the catalog in the config to match + >>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ === The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again + >>> [CLI] bundle plan --output json { "resources.catalogs.catalog1": { @@ -130,6 +134,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } === The catalog's create fails because the renamed catalog has its name + >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Error: cannot create resources.catalogs.catalog1: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists (400 CATALOG_ALREADY_EXISTS) diff --git a/acceptance/bundle/resources/catalogs/rename_with_comment/output.txt b/acceptance/bundle/resources/catalogs/rename_with_comment/output.txt index 0359d87b31a..0b82c1e0697 100644 --- a/acceptance/bundle/resources/catalogs/rename_with_comment/output.txt +++ b/acceptance/bundle/resources/catalogs/rename_with_comment/output.txt @@ -1,5 +1,6 @@ === Deploy a catalog with a comment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created catalogs.catalog1 @@ -7,6 +8,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Rename the catalog and change its comment in one deploy: both go in one PATCH + >>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ >>> update_file.py databricks.yml comment: c1 comment: c2 @@ -29,6 +31,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Both landed, so the redeploy is a no-op + >>> [CLI] catalogs get test_catalog_renamed_[UNIQUE_NAME] { "name": "test_catalog_renamed_[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/output.txt b/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/output.txt index b6c56a0ee49..0997557db9b 100644 --- a/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/output.txt +++ b/acceptance/bundle/resources/schemas/destroy_after_rename_out_of_band/output.txt @@ -1,5 +1,6 @@ === Deploy a schema with a volume in it + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -8,6 +9,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Rename the schema out of band, and the config to match + >>> [CLI] schemas update main.test-schema-[UNIQUE_NAME] --new-name test-schema-renamed-[UNIQUE_NAME] { "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", @@ -17,6 +19,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml test-schema- test-schema-renamed- === Destroy + >>> [CLI] bundle destroy --auto-approve All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default @@ -25,6 +28,7 @@ Destroy: 0 deleted >>> print_requests.py //unity-catalog === The renamed schema is still there + >>> [CLI] schemas get main.test-schema-renamed-[UNIQUE_NAME] { "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/schemas/foreign_at_recorded_name/output.txt b/acceptance/bundle/resources/schemas/foreign_at_recorded_name/output.txt index abe10612e29..fcfa9eb6c55 100644 --- a/acceptance/bundle/resources/schemas/foreign_at_recorded_name/output.txt +++ b/acceptance/bundle/resources/schemas/foreign_at_recorded_name/output.txt @@ -1,5 +1,6 @@ === Deploy a schema + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -7,6 +8,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Out of band, rename the schema away, and create another one under its name + >>> [CLI] schemas update main.test-schema-[UNIQUE_NAME] --new-name test-schema-moved-[UNIQUE_NAME] { "full_name": "main.test-schema-moved-[UNIQUE_NAME]", @@ -14,6 +16,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === The plan takes the other schema for the bundle's, though its schema_id differs, and plans to update it + >>> [CLI] bundle plan --output json { "resources.schemas.schema1": { @@ -53,6 +56,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Destroy deletes the other schema + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.schemas.schema1 @@ -77,6 +81,7 @@ Destroy: 1 deleted Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. === The bundle's own schema is left behind + >>> [CLI] schemas get main.test-schema-moved-[UNIQUE_NAME] { "full_name": "main.test-schema-moved-[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt b/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt index b8595d0fa84..2a4a2e8da0b 100644 --- a/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt @@ -1,5 +1,6 @@ === Deploy a schema with a volume in it + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -8,6 +9,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Rename the schema and change its storage_root, which recreates it. The CLI is killed on the volume's recreate, after the schema's + >>> update_file.py databricks.yml test-schema- test-schema-renamed- >>> update_file.py databricks.yml s3://mybucket/a s3://mybucket/b @@ -46,6 +48,7 @@ Exit code: [KILLED] } === The volume went with the old schema. Nothing with its volume_id is at its new path, so the plan creates it + >>> [CLI] bundle plan --output json { "resources.schemas.schema1": { @@ -111,6 +114,7 @@ Exit code: [KILLED] } === The next deploy creates a new volume + >>> [CLI] bundle deploy --force-lock Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created volumes.volume1 @@ -136,6 +140,7 @@ Resources: 1 created, 0 changed, 0 deleted, 1 unchanged } === Redeploy is a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt b/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt index 36e38a1c807..5e1ad2e7cd1 100644 --- a/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt @@ -1,5 +1,6 @@ === Deploy a schema with a volume in it + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -8,9 +9,11 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Out of band, drop the schema and create another one under a new name, with a volume of the same name + >>> [CLI] schemas delete main.test-schema-[UNIQUE_NAME] --force === With the config renamed to match, the plan finds a schema and a volume at the new names, but their ids differ, so it plans creates + >>> update_file.py databricks.yml test-schema- test-schema-renamed- >>> print_state.py diff --git a/acceptance/bundle/resources/schemas/rename/output.txt b/acceptance/bundle/resources/schemas/rename/output.txt index 57cab932c15..23a2d71f771 100644 --- a/acceptance/bundle/resources/schemas/rename/output.txt +++ b/acceptance/bundle/resources/schemas/rename/output.txt @@ -1,5 +1,6 @@ === Deploy the schema + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -18,9 +19,11 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Rename the schema + >>> update_file.py databricks.yml test-schema- test-schema-renamed- === The plan renames the schema in place instead of recreating it + >>> [CLI] bundle plan --output json { "resources.schemas.schema1": { @@ -65,6 +68,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Deploy renames the schema without a deletion prompt + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated schemas.schema1 @@ -83,6 +87,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === The schema is only reachable under the new name, with its original schema_id + >>> musterr [CLI] schemas get main.test-schema-[UNIQUE_NAME] Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. @@ -95,6 +100,7 @@ Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. } === The state and summary track the new name + >>> [CLI] bundle summary --output json { "id": "main.test-schema-renamed-[UNIQUE_NAME]", @@ -102,6 +108,7 @@ Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. } === Redeploy is a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/schemas/rename_case_only/output.txt b/acceptance/bundle/resources/schemas/rename_case_only/output.txt index cdeaaeab649..19e68a6609a 100644 --- a/acceptance/bundle/resources/schemas/rename_case_only/output.txt +++ b/acceptance/bundle/resources/schemas/rename_case_only/output.txt @@ -1,5 +1,6 @@ === Deploy a schema with a lowercase name + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -7,6 +8,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Change only the case of the name. UC lowercases names, so it is the same name + >>> update_file.py databricks.yml test_schema_ Test_Schema_ >>> [CLI] bundle plan --output json @@ -75,6 +77,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Redeploy + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt index 27039e2150b..edf4f618ffb 100644 --- a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt +++ b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt @@ -1,5 +1,6 @@ === Deploy a schema with a volume in it + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -8,6 +9,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Rename the schema and change the volume's comment. The CLI is killed on the volume's update, after the schema's rename is saved + >>> update_file.py databricks.yml test-schema- test-schema-renamed- >>> update_file.py databricks.yml comment: c1 comment: c2 @@ -30,6 +32,7 @@ Exit code: [KILLED] } === The volume is gone from its recorded path, so the plan looks for it at its path in config + >>> print_state.py { "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", @@ -139,6 +142,7 @@ Exit code: [KILLED] } === The next deploy updates the volume at its new path + >>> [CLI] bundle deploy --force-lock Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated volumes.volume1 @@ -162,6 +166,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } === Redeploy is a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt b/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt index d648ff373ce..7a1fbc10831 100644 --- a/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt @@ -1,5 +1,6 @@ === Deploy a schema with a volume in it + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -8,6 +9,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Rename the schema out of band. UC moves the volume with it + >>> [CLI] schemas update main.test-schema-[UNIQUE_NAME] --new-name test-schema-renamed-[UNIQUE_NAME] { "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", @@ -15,6 +17,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === With the config unchanged, the plan creates the schema and the volume again under their configured names + >>> [CLI] bundle plan create schemas.schema1 create volumes.volume1 @@ -22,6 +25,7 @@ create volumes.volume1 Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged === With the config renamed to match, the plan finds both at their new names, with the same schema_id and volume_id + >>> update_file.py databricks.yml test-schema- test-schema-renamed- >>> [CLI] bundle plan --output json @@ -112,6 +116,7 @@ Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged } === Deploy only records the new names in the state + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Files: 3 uploaded, 0 deleted @@ -142,6 +147,7 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged } === Redeploy is a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/output.txt b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/output.txt index 604d46a7586..8bc5e7020f0 100644 --- a/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/output.txt +++ b/acceptance/bundle/resources/schemas/rename_out_of_band_old_state/output.txt @@ -1,5 +1,6 @@ === Deploy a schema with a volume in it + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -8,6 +9,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Drop schema_id and volume_id from the state, as a CLI from before they were recorded wrote it + >>> jq .state | map_values(.state | {schema_id, volume_id} | with_entries(select(.value != null))) .databricks/bundle/default/resources.json { "resources.schemas.schema1": {}, @@ -15,6 +17,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === A deploy with nothing else to change records them, with a state-only update + >>> [CLI] bundle plan --output json { "resources.schemas.schema1": { @@ -102,6 +105,7 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged } === Rename the schema out of band, and the config to match. The plan finds both by their ids + >>> [CLI] schemas update main.test-schema-[UNIQUE_NAME] --new-name test-schema-renamed-[UNIQUE_NAME] { "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", @@ -205,6 +209,7 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //unity-catalog === Redeploy is a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/schemas/rename_probe_forbidden/output.txt b/acceptance/bundle/resources/schemas/rename_probe_forbidden/output.txt index 87df5227de4..485dbfc0976 100644 --- a/acceptance/bundle/resources/schemas/rename_probe_forbidden/output.txt +++ b/acceptance/bundle/resources/schemas/rename_probe_forbidden/output.txt @@ -1,5 +1,6 @@ === Deploy a schema + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created schemas.schema1 @@ -7,6 +8,7 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Delete the schema out of band and rename it in the config. Reading the new name is forbidden, so the plan only warns and creates the schema + >>> [CLI] schemas delete main.myschema >>> update_file.py databricks.yml name: myschema name: myschema2 diff --git a/acceptance/bundle/resources/schemas/rename_then_change/output.txt b/acceptance/bundle/resources/schemas/rename_then_change/output.txt index a3f6f27efc7..8218075b830 100644 --- a/acceptance/bundle/resources/schemas/rename_then_change/output.txt +++ b/acceptance/bundle/resources/schemas/rename_then_change/output.txt @@ -1,5 +1,6 @@ === Deploy two schemas and a volume in the first one + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -38,6 +39,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } === Rename the volume's schema: the volume is re-keyed without a request + >>> update_file.py databricks.yml test-schema1- test-schema1-renamed- >>> [CLI] bundle deploy @@ -58,6 +60,7 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged } === After the rename, update, update_id and recreate of the volume all reach it at its new path + >>> update_file.py databricks.yml comment: c1 comment: c2 >>> [CLI] bundle deploy @@ -125,6 +128,7 @@ Resources: 1 created, 0 changed, 1 deleted, 2 unchanged } === Rename the volume's new schema: the volume is re-keyed again + >>> update_file.py databricks.yml test-schema2- test-schema2-renamed- >>> [CLI] bundle deploy @@ -145,6 +149,7 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged } === Update, update_id and recreate work again, moving the volume back to the first schema + >>> update_file.py databricks.yml comment: c2 comment: c3 >>> [CLI] bundle deploy @@ -212,6 +217,7 @@ Resources: 1 created, 0 changed, 1 deleted, 2 unchanged } === Rename the first schema back: the volume is re-keyed to its original path + >>> update_file.py databricks.yml test-schema1-renamed- test-schema1- >>> [CLI] bundle deploy @@ -232,6 +238,7 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged } === The state tracks the current paths and the plan is a no-op + >>> [CLI] bundle summary --output json { "schemas": { diff --git a/acceptance/bundle/resources/schemas/rename_with_comment/output.txt b/acceptance/bundle/resources/schemas/rename_with_comment/output.txt index f685df69adc..a61cb0a450d 100644 --- a/acceptance/bundle/resources/schemas/rename_with_comment/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_comment/output.txt @@ -1,5 +1,6 @@ === Deploy a schema with a comment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -7,6 +8,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Rename the schema and change its comment in one deploy: both go in one PATCH + >>> update_file.py databricks.yml test-schema- test-schema-renamed- >>> update_file.py databricks.yml comment: c1 comment: c2 @@ -29,6 +31,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Both landed, so the redeploy is a no-op + >>> [CLI] schemas get main.test-schema-renamed-[UNIQUE_NAME] { "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt index bb058ef40fd..01484539e0b 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -1,5 +1,6 @@ === Deploy a schema with a volume in it + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -28,9 +29,11 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Rename the schema + >>> update_file.py databricks.yml test-schema- test-schema-renamed- === UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it + >>> [CLI] bundle plan --output json { "resources.schemas.schema1": { @@ -116,6 +119,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Deploy renames the schema. UC moves the volume with it, so the volume gets no request + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated schemas.schema1 @@ -134,6 +138,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } === The old schema is gone and the volume is under the new one, with its original volume_id + >>> musterr [CLI] schemas get main.test-schema-[UNIQUE_NAME] Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. @@ -145,10 +150,12 @@ Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. } === The state tracks the new path + >>> [CLI] bundle summary --output json "main.test-schema-renamed-[UNIQUE_NAME].myvolume" === Redeploy is a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt index 8807308a705..f9ee1f48e1c 100644 --- a/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt @@ -1,5 +1,6 @@ === Deploy a schema, a volume in it and a job that reads the volume's full_name and id + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created jobs.job1 @@ -25,9 +26,11 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged ] === Rename the schema + >>> update_file.py databricks.yml test-schema- test-schema-renamed- === The volume's full_name and id change with the schema, so the job is updated too + >>> [CLI] bundle plan update jobs.job1 update schemas.schema1 @@ -35,6 +38,7 @@ update schemas.schema1 Plan: 0 to add, 2 to change, 0 to delete, 1 unchanged === Deploy passes the volume's new full_name and id to the job + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated jobs.job1 @@ -70,6 +74,7 @@ Resources: 0 created, 2 changed, 0 deleted, 1 unchanged ] === Redeploy is a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged diff --git a/acceptance/bundle/resources/volumes/move_from_renamed_schema/output.txt b/acceptance/bundle/resources/volumes/move_from_renamed_schema/output.txt index ccee5240bef..c265ef49c8d 100644 --- a/acceptance/bundle/resources/volumes/move_from_renamed_schema/output.txt +++ b/acceptance/bundle/resources/volumes/move_from_renamed_schema/output.txt @@ -1,5 +1,6 @@ === Deploy two schemas and a volume in the first one + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -9,6 +10,7 @@ Files: 2 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Rename the first schema and move the volume to the second one in one deploy. Moving the volume recreates it + >>> update_file.py databricks.yml test-schema1- test-schema1-renamed- >>> update_file.py databricks.yml schemas.schema1.name schemas.schema2.name @@ -160,16 +162,19 @@ Resources: 1 created, 1 changed, 1 deleted, 1 unchanged } === The volume is in the second schema + >>> [CLI] volumes read main.test-schema2-[UNIQUE_NAME].myvolume { "full_name": "main.test-schema2-[UNIQUE_NAME].myvolume" } === The old volume, moved with the renamed schema, was deleted there + >>> musterr [CLI] volumes read main.test-schema1-renamed-[UNIQUE_NAME].myvolume Error: Volume 'main.test-schema1-renamed-[UNIQUE_NAME].myvolume' does not exist. === Redeploy is a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged diff --git a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/output.txt b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/output.txt index 82200b34596..86f99cf6f9b 100644 --- a/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/output.txt +++ b/acceptance/bundle/resources/volumes/recreate_in_renamed_schema/output.txt @@ -1,5 +1,6 @@ === Deploy a schema with a managed volume in it + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -8,6 +9,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Rename the schema and change the volume's type in one deploy. The type change recreates the volume + >>> update_file.py databricks.yml test-schema- test-schema-renamed- >>> update_file.py databricks.yml volume_type: MANAGED volume_type: EXTERNAL @@ -153,6 +155,7 @@ Resources: 1 created, 1 changed, 1 deleted, 0 unchanged } === Redeploy is a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt b/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt index b8b36ce7130..85fb3ea1d62 100644 --- a/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt @@ -1,5 +1,6 @@ === Deploy a schema with a volume in it + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -8,9 +9,11 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Out of band, delete the volume and create another one under a new name + >>> [CLI] volumes delete main.test-schema-[UNIQUE_NAME].myvolume === With the config renamed to match, the plan finds a volume at the new name, but its volume_id differs, so it plans a create + >>> update_file.py databricks.yml name: myvolume name: myvolume2 >>> print_state.py diff --git a/acceptance/bundle/resources/volumes/rename_case_only/output.txt b/acceptance/bundle/resources/volumes/rename_case_only/output.txt index 3c1b9113c16..4627fe24e30 100644 --- a/acceptance/bundle/resources/volumes/rename_case_only/output.txt +++ b/acceptance/bundle/resources/volumes/rename_case_only/output.txt @@ -1,5 +1,6 @@ === Deploy a volume with a lowercase name + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -8,6 +9,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Change only the case of the name. UC lowercases names, so it is the same name + >>> update_file.py databricks.yml name: myvolume name: MyVolume >>> [CLI] bundle plan --output json @@ -104,10 +106,12 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } === Redeploy + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === Rename the schema. The volume is re-keyed without a request. Its recorded ID keeps the config's casing, which UC reads case-insensitively + >>> update_file.py databricks.yml test_schema_ test_schema_renamed_ >>> [CLI] bundle deploy diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt index e8fd1ecfdde..f5d7df1a699 100644 --- a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt @@ -1,5 +1,6 @@ === Deploy a catalog with a schema and a volume in it + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created catalogs.catalog1 @@ -9,11 +10,13 @@ Files: 2 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Rename the catalog and the schema in one deploy + >>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ >>> update_file.py databricks.yml name: myschema name: myschema2 === Both of the volume's parents are renamed, so both its catalog_name and schema_name follow them + >>> [CLI] bundle plan --output json { "resources.catalogs.catalog1": { @@ -146,6 +149,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } === Deploy renames the catalog, then the schema under it. UC moves the volume both times, so it gets no request + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated catalogs.catalog1 @@ -174,6 +178,7 @@ Resources: 0 created, 2 changed, 0 deleted, 1 unchanged } === The schema and the volume are under the new names, with their original ids + >>> [CLI] schemas get test_catalog_renamed_[UNIQUE_NAME].myschema2 { "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2", @@ -200,6 +205,7 @@ Resources: 0 created, 2 changed, 0 deleted, 1 unchanged } === Redeploy is a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt b/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt index f62a0c3920a..1926f3aafab 100644 --- a/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt @@ -1,5 +1,6 @@ === Deploy a schema with a volume in it + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -8,6 +9,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Rename the volume out of band + >>> [CLI] volumes update main.test-schema-[UNIQUE_NAME].myvolume --new-name myvolume2 { "full_name": "main.test-schema-[UNIQUE_NAME].myvolume2", @@ -15,12 +17,14 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === With the config unchanged, the plan creates the volume again under its configured name + >>> [CLI] bundle plan create volumes.volume1 Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged === With the config renamed to match, the plan finds the volume at its new name, with the same volume_id + >>> update_file.py databricks.yml name: myvolume name: myvolume2 >>> [CLI] bundle plan --output json @@ -96,6 +100,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged } === Deploy only records the new name in the state + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Files: 3 uploaded, 0 deleted @@ -113,6 +118,7 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged "main.test-schema-[UNIQUE_NAME].myvolume2" === Redeploy is a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged From dec3b6f5243cfc5bc07d4152817c48fa9f180ef0 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 12 Nov 2026 12:11:32 +0100 Subject: [PATCH 63/75] acc: restore the recorded schema_id/volume_id in goldens the rebase reset to main's Co-authored-by: Isaac --- .../empty-resource-id/update/output.txt | 1 + .../bundle/dms/declined-deploy/output.txt | 9 ++++---- .../dms/deployment-metadata-change/output.txt | 6 +++--- acceptance/bundle/dms/empty-grants/output.txt | 9 ++++---- .../bundle/dms/failed-recreate/output.txt | 11 +++++----- .../operation-upload-fails-once/output.txt | 14 +++++++------ .../bundle/dms/successful-recreate/output.txt | 18 +++++++++------- .../grants/schemas/change_name/output.txt | 21 ++++++------------- .../schemas/comment_out_of_band/output.txt | 6 ++++++ .../drift/managed_properties/output.txt | 6 ++++++ .../volumes/comment_out_of_band/output.txt | 6 ++++++ 11 files changed, 62 insertions(+), 45 deletions(-) diff --git a/acceptance/bundle/deploy/empty-resource-id/update/output.txt b/acceptance/bundle/deploy/empty-resource-id/update/output.txt index ada20dd341a..c4b1bfe2c64 100644 --- a/acceptance/bundle/deploy/empty-resource-id/update/output.txt +++ b/acceptance/bundle/deploy/empty-resource-id/update/output.txt @@ -25,6 +25,7 @@ Files: 0 uploaded, 0 deleted "catalog_name": "main", "name": "myvolume", "schema_name": "myschema", + "volume_id": "[UUID]", "volume_type": "MANAGED" } } diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index f8b66625dcc..d9c4c81e40c 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -48,7 +48,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_declined_deploy_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_declined_deploy_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID][0]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -66,7 +66,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][1]", "serial": 1, "features": { "deployment_history_shadow": {} @@ -90,7 +90,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged { "state_version": 3, "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][1]", "serial": 1, "features": { "deployment_history_shadow": {} @@ -100,7 +100,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "__id__": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", "state": { "catalog_name": "main", - "name": "dms_declined_deploy_schema_[UNIQUE_NAME]" + "name": "dms_declined_deploy_schema_[UNIQUE_NAME]", + "schema_id": "[UUID][0]" } } } diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 6cae20bc707..ad325a6903f 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -48,7 +48,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_metadata_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_metadata_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_metadata_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID][0]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -66,7 +66,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][1]", "serial": 1, "features": { "deployment_history_shadow": {} @@ -128,7 +128,7 @@ A common practice is to use a username or principal name in this path, i.e. use { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][1]", "serial": 1, "features": { "deployment_history_shadow": {} diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index 32bda513d5f..1907776537d 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -53,7 +53,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_empty_grants_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_empty_grants_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_empty_grants_[UNIQUE_NAME]\",\"schema_id\":\"[UUID][0]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -139,7 +139,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][1]", "serial": 2, "features": { "deployment_history_shadow": {} @@ -166,7 +166,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged { "state_version": 3, "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][1]", "serial": 2, "features": { "deployment_history_shadow": {} @@ -176,7 +176,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "__id__": "main.dms_empty_grants_[UNIQUE_NAME]", "state": { "catalog_name": "main", - "name": "dms_empty_grants_[UNIQUE_NAME]" + "name": "dms_empty_grants_[UNIQUE_NAME]", + "schema_id": "[UUID][0]" } } } diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 88890f0e2a9..c2f8ff329db 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -48,7 +48,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_failed_recreate_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_recreate_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_recreate_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID][0]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -66,7 +66,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][1]", "serial": 1, "features": { "deployment_history_shadow": {} @@ -93,7 +93,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged { "state_version": 3, "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][1]", "serial": 1, "features": { "deployment_history_shadow": {} @@ -104,7 +104,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state": { "catalog_name": "main", "comment": "v1", - "name": "dms_failed_recreate_schema_[UNIQUE_NAME]" + "name": "dms_failed_recreate_schema_[UNIQUE_NAME]", + "schema_id": "[UUID][0]" } } } @@ -192,7 +193,7 @@ Files: 2 uploaded, 0 deleted { "state_version": 3, "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][1]", "serial": 2, "features": { "deployment_history_shadow": {} diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index 6f5e558dbc9..aeb97ce4cdb 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -13,7 +13,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][0]", "serial": 1, "features": { "deployment_history_shadow": {} @@ -40,7 +40,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged { "state_version": 3, "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][0]", "serial": 1, "features": { "deployment_history_shadow": {} @@ -51,7 +51,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state": { "catalog_name": "main", "comment": "v1", - "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" + "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]", + "schema_id": "[UUID][1]" } } } @@ -103,7 +104,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged { "state_version": 3, "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][0]", "serial": 2, "features": { "deployment_history_shadow": {} @@ -114,7 +115,8 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "state": { "catalog_name": "dms_other_[UNIQUE_NAME]", "comment": "v1", - "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" + "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]", + "schema_id": "[UUID][2]" } } } @@ -161,7 +163,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "error_message": "", "resource_id": "dms_other_[UNIQUE_NAME].dms_upload_fails_once_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_upload_fails_once_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_upload_fails_once_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID][2]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 3a414ec1815..225bea93a80 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -48,7 +48,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_successful_recreate_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID][0]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -66,7 +66,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][1]", "serial": 1, "features": { "deployment_history_shadow": {} @@ -93,7 +93,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged { "state_version": 3, "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][1]", "serial": 1, "features": { "deployment_history_shadow": {} @@ -104,7 +104,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state": { "catalog_name": "main", "comment": "v1", - "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]", + "schema_id": "[UUID][0]" } } } @@ -164,7 +165,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "error_message": "", "resource_id": "dms_other_[UNIQUE_NAME].dms_successful_recreate_schema_[UNIQUE_NAME]", "sequence_id": 1, - "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID][2]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -182,7 +183,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][1]", "serial": 2, "features": { "deployment_history_shadow": {} @@ -209,7 +210,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged { "state_version": 3, "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][1]", "serial": 2, "features": { "deployment_history_shadow": {} @@ -220,7 +221,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state": { "catalog_name": "dms_other_[UNIQUE_NAME]", "comment": "v1", - "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]", + "schema_id": "[UUID][2]" } } } diff --git a/acceptance/bundle/resources/grants/schemas/change_name/output.txt b/acceptance/bundle/resources/grants/schemas/change_name/output.txt index 1cc43885cdf..b7e1feb380d 100644 --- a/acceptance/bundle/resources/grants/schemas/change_name/output.txt +++ b/acceptance/bundle/resources/grants/schemas/change_name/output.txt @@ -57,28 +57,19 @@ schemas schema1.grants id='schema/main.test-schema-[UNIQUE_NAME]' >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... - -This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: - recreate resources.schemas.schema1 -Recreated schemas.schema1 +Updated schemas.schema1 Updated schemas.schema1.grants Files: 1 uploaded, 0 deleted -Resources: 1 created, 1 changed, 1 deleted, 0 unchanged +Resources: 0 created, 2 changed, 0 deleted, 0 unchanged >>> print_requests.py //unity-catalog { - "method": "DELETE", + "method": "PATCH", "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", - "q": { - "force": "true" - } -} -{ - "method": "POST", - "path": "/api/2.1/unity-catalog/schemas", "body": { - "catalog_name": "main", - "name": "test-schema-renamed-[UNIQUE_NAME]" + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" } } { diff --git a/acceptance/bundle/resources/schemas/comment_out_of_band/output.txt b/acceptance/bundle/resources/schemas/comment_out_of_band/output.txt index 7bcd7ba6aad..19c218a806e 100644 --- a/acceptance/bundle/resources/schemas/comment_out_of_band/output.txt +++ b/acceptance/bundle/resources/schemas/comment_out_of_band/output.txt @@ -16,6 +16,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "comment": { "action": "update", "remote": "set outside the bundle" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" } } diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt index f490e82091b..29000050d16 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt @@ -28,6 +28,12 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "unity.catalog.managed.delta.defaults.delta.parquet.format.version.afe.internal": "2.12.0", "unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged": "true" } + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" } } diff --git a/acceptance/bundle/resources/volumes/comment_out_of_band/output.txt b/acceptance/bundle/resources/volumes/comment_out_of_band/output.txt index 651e7be09bd..21886a1c9e9 100644 --- a/acceptance/bundle/resources/volumes/comment_out_of_band/output.txt +++ b/acceptance/bundle/resources/volumes/comment_out_of_band/output.txt @@ -22,6 +22,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "action": "skip", "reason": "immutable_backend_value", "remote": "s3://[METASTORE_NAME]/metastore/[UUID][0]/volumes/[UUID][1]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][1]", + "remote": "[UUID][1]" } } From e56021ea894fa3e2839923ff68b2c1ffa1853395 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 12 Nov 2026 14:12:09 +0100 Subject: [PATCH 64/75] direct: collect parent renames only for types that can follow one, with IDFromState (review C8) Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 2817f29f44e..d17a816c83c 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -267,7 +267,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks // Collected before resolveReferences, which drops the references it resolves. Only a resource // that can follow a rename needs them (see followsParentRename). var parentRenames map[string]*deployplan.ChangeDesc - if adapter.HasDoUpdateWithID() { + if adapter.HasDoUpdateWithID() && adapter.HasIDFromState() { parentRenames, err = b.parentRenames(resourceKey) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) From 232956fb09e11613e2924123f9c70ff037a3bcd8 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 12 Nov 2026 14:12:12 +0100 Subject: [PATCH 65/75] statemgmt: say which internal ID the migration's plan check supplies (review C10) Co-authored-by: Isaac --- bundle/statemgmt/direct_migration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index 64c78cf10df..11a0fc452c1 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -108,7 +108,7 @@ func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) { if err := json.Unmarshal(raw, &data); err != nil { return false, fmt.Errorf("parsing migrated state: %w", err) } - // The terraform state has no internal IDs (schema_id, volume_id); the plan check read them. + // The terraform state has schema_id but no volume_id (see BuildStateFromTF); the plan check read it. if err := direct.RecordMissingInternalIDs(plan, &data); err != nil { return false, fmt.Errorf("recording internal ids in migrated state: %w", err) } From 1e2e3bbb686cc4234ce53c6020a5785350a3fffc Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 12 Nov 2026 14:16:43 +0100 Subject: [PATCH 66/75] direct: a bind whose only plan is a state-only update needs no confirmation (review C2) A bind records no internal ID, so the bound volume or schema plans a state-only update; it changes nothing remotely. The volume bind test now binds without --auto-approve. Co-authored-by: Isaac --- acceptance/bundle/deployment/bind/volume/output.txt | 2 +- acceptance/bundle/deployment/bind/volume/script | 2 +- bundle/direct/bind.go | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/deployment/bind/volume/output.txt b/acceptance/bundle/deployment/bind/volume/output.txt index 89509ef00a8..4ccca13ddf2 100644 --- a/acceptance/bundle/deployment/bind/volume/output.txt +++ b/acceptance/bundle/deployment/bind/volume/output.txt @@ -7,7 +7,7 @@ === Create a pre-defined volume: ->>> [CLI] bundle deployment bind volume1 main.test-schema-[UNIQUE_NAME].volume-[UNIQUE_NAME] --auto-approve +>>> [CLI] bundle deployment bind volume1 main.test-schema-[UNIQUE_NAME].volume-[UNIQUE_NAME] Successfully bound volume with an id 'main.test-schema-[UNIQUE_NAME].volume-[UNIQUE_NAME]' Run 'bundle deploy' to deploy changes to your workspace diff --git a/acceptance/bundle/deployment/bind/volume/script b/acceptance/bundle/deployment/bind/volume/script index dee5352a837..2656435ca94 100644 --- a/acceptance/bundle/deployment/bind/volume/script +++ b/acceptance/bundle/deployment/bind/volume/script @@ -17,7 +17,7 @@ cleanupRemoveVolume() { } trap cleanupRemoveVolume EXIT -trace $CLI bundle deployment bind volume1 "${VOLUME_FULL_NAME}" --auto-approve +trace $CLI bundle deployment bind volume1 "${VOLUME_FULL_NAME}" trace $CLI bundle deploy diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 7fbf7ede662..73852cdde00 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -218,7 +218,9 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac entry = plan.Plan[resourceKey] if entry != nil { result.Action = entry.Action - result.HasChanges = result.Action != deployplan.Skip && result.Action != deployplan.Undefined + // A state-only entry only records something in the state (such as an internal ID a bind + // does not write), so it changes nothing remotely. + result.HasChanges = result.Action != deployplan.Skip && result.Action != deployplan.Undefined && !entry.IsStateOnly() } return result, nil From 2b719a89e42c811e7d63f4f97a38451172596fe8 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 12 Nov 2026 14:20:26 +0100 Subject: [PATCH 67/75] direct: references to a state-only update read the remote state (review C3) On the first deploy after upgrading, a volume or schema whose state lacks its internal ID plans a state-only update. References to its remote-only fields (storage_location, volume_id) were delayed, so a job reading them was updated for nothing. A state-only update changes nothing remotely, like a skip. Co-authored-by: Isaac --- .../volumes/old_state_ref/databricks.yml.tmpl | 23 +++++++++ .../volumes/old_state_ref/out.test.toml | 2 + .../volumes/old_state_ref/output.txt | 48 +++++++++++++++++++ .../resources/volumes/old_state_ref/script | 27 +++++++++++ .../resources/volumes/old_state_ref/test.toml | 11 +++++ bundle/direct/bundle_plan.go | 3 ++ 6 files changed, 114 insertions(+) create mode 100644 acceptance/bundle/resources/volumes/old_state_ref/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/volumes/old_state_ref/out.test.toml create mode 100644 acceptance/bundle/resources/volumes/old_state_ref/output.txt create mode 100644 acceptance/bundle/resources/volumes/old_state_ref/script create mode 100644 acceptance/bundle/resources/volumes/old_state_ref/test.toml diff --git a/acceptance/bundle/resources/volumes/old_state_ref/databricks.yml.tmpl b/acceptance/bundle/resources/volumes/old_state_ref/databricks.yml.tmpl new file mode 100644 index 00000000000..214c04d32d8 --- /dev/null +++ b/acceptance/bundle/resources/volumes/old_state_ref/databricks.yml.tmpl @@ -0,0 +1,23 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume + jobs: + job1: + name: test-job-$UNIQUE_NAME + parameters: + - name: volume_full_name + default: ${resources.volumes.volume1.full_name} + - name: volume_uuid + default: ${resources.volumes.volume1.volume_id} + - name: storage_location + default: ${resources.volumes.volume1.storage_location} diff --git a/acceptance/bundle/resources/volumes/old_state_ref/out.test.toml b/acceptance/bundle/resources/volumes/old_state_ref/out.test.toml new file mode 100644 index 00000000000..f68b4a89c36 --- /dev/null +++ b/acceptance/bundle/resources/volumes/old_state_ref/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/volumes/old_state_ref/output.txt b/acceptance/bundle/resources/volumes/old_state_ref/output.txt new file mode 100644 index 00000000000..1cbda338d87 --- /dev/null +++ b/acceptance/bundle/resources/volumes/old_state_ref/output.txt @@ -0,0 +1,48 @@ + +=== Deploy a volume and a job that reads fields of it the config does not set + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created jobs.job1 +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Drop schema_id and volume_id from the state, as a CLI from before they were recorded wrote it + +=== The volume only records its id; the job still reads the volume's fields and is unchanged + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 3 unchanged + +>>> print_requests.py //unity-catalog //jobs + +=== Redeploy is a no-op + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.job1 + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/volumes/old_state_ref/script b/acceptance/bundle/resources/volumes/old_state_ref/script new file mode 100644 index 00000000000..41fbf8aea59 --- /dev/null +++ b/acceptance/bundle/resources/volumes/old_state_ref/script @@ -0,0 +1,27 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +STATE=.databricks/bundle/default/resources.json + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog //jobs &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a volume and a job that reads fields of it the config does not set" +trace $CLI bundle deploy +print_requests.py //unity-catalog //jobs &> LOG.deploy_requests + +title "Drop schema_id and volume_id from the state, as a CLI from before they were recorded wrote it" +jq 'del(.state["resources.schemas.schema1"].state.schema_id, .state["resources.volumes.volume1"].state.volume_id)' "$STATE" > tmp.json +mv tmp.json "$STATE" + +title "The volume only records its id; the job still reads the volume's fields and is unchanged" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //unity-catalog //jobs + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/volumes/old_state_ref/test.toml b/acceptance/bundle/resources/volumes/old_state_ref/test.toml new file mode 100644 index 00000000000..e0f4bb72822 --- /dev/null +++ b/acceptance/bundle/resources/volumes/old_state_ref/test.toml @@ -0,0 +1,11 @@ +Cloud = true + +# The test rewrites the deployment state by hand, which the recording service refuses because it +# no longer matches the deployment it has recorded, so run without deployment history. +EnvMatrix.DMS = [""] + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index d17a816c83c..ae6bed15e36 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -1405,7 +1405,10 @@ func (b *DeploymentBundle) LookupReferencePreDeploy(ctx context.Context, path *s // like an AIP name derived from the ID (FieldIsStableOutput). The latter lets // ${resources.X.name} resolve during an in-place update instead of delaying, // which would otherwise recreate dependents that reference it as their parent. + // A state-only update changes nothing remotely either (unlike a state-only + // update_id, whose remote state predates its parent's rename). canReadRemoteCache := targetAction == deployplan.Skip || + (targetAction == deployplan.Update && targetEntry.StateOnly) || (targetAction.KeepsID() && (adapter.FieldTriggersRecreate(fieldPath) || adapter.FieldIsStableOutput(fieldPath))) if configValidErr != nil && remoteValidErr == nil { From abc90079c4468b47ddf23b0f811836e7b9dcba33 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 12 Nov 2026 14:23:55 +0100 Subject: [PATCH 68/75] direct: re-key only the field that references the renamed parent (review C4) addParentMoves rewrote every changed field whose old value equaled the parent's old name, so a volume named like its schema got a wrong ID for the recreate's delete. It now rewrites a field only if it references a resource of the parent's type, recorded before the references are resolved. Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 38 +++++++++++++++++++++++++++++++ bundle/direct/bundle_plan_test.go | 22 ++++++++++++++++-- bundle/direct/pkg.go | 4 ++++ 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index ae6bed15e36..5ae6f831cca 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -275,6 +275,13 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks } } + if adapter.HasIDFromState() { + if err := b.recordRefTypes(resourceKey); err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } + } + // Process all references in the resource using Refs map // Refs maps path inside resource to references e.g. "${resources.jobs.foo.id} ${resources.jobs.foo.name}" if !b.resolveReferences(ctx, resourceKey, entry, errorPrefix, true) { @@ -798,6 +805,7 @@ func (b *DeploymentBundle) addParentMoves(plan *deployplan.Plan) error { return fmt.Errorf("%s: interpreting state: %w", resourceKey, err) } + refTypes, _ := b.refTypes.Load(resourceKey) var parents []deployplan.DependsOnEntry for _, dep := range dbentry.DependsOn { rename, err := b.dependencyRename(dep.Label) @@ -807,7 +815,13 @@ func (b *DeploymentBundle) addParentMoves(plan *deployplan.Plan) error { if rename == nil { continue } + parentType := config.GetResourceTypeFromKey(dep.Node) for pathString, ch := range entry.Changes { + // Only a field referencing a resource of the parent's type followed it (schema_name for a + // schema); another one that merely had the same value, like the resource's own name, did not. + if types, ok := refTypes.(map[string]string); !ok || types[pathString] != parentType { + continue + } if !structdiff.IsEqual(ch.Old, rename.Old) { continue } @@ -842,6 +856,30 @@ func (b *DeploymentBundle) addParentMoves(plan *deployplan.Plan) error { return nil } +// recordRefTypes records, for each pure-reference field of resourceKey, the type of the resource it +// references, before resolveReferences drops the references (see addParentMoves). +func (b *DeploymentBundle) recordRefTypes(resourceKey string) error { + sv, ok := b.StateCache.Load(resourceKey) + if !ok { + return fmt.Errorf("internal error: no state cache entry found for %q", resourceKey) + } + types := make(map[string]string, len(sv.Refs)) + for fieldPath, refString := range sv.Refs { + ref, ok := dynvar.NewRef(dyn.V(refString)) + if !ok || !ref.IsPure() { + continue + } + targetPath, err := structpath.ParsePath(ref.References()[0]) + if err != nil { + return fmt.Errorf("cannot parse reference %q: %w", refString, err) + } + targetKey, _ := splitResourcePath(targetPath) + types[fieldPath] = config.GetResourceTypeFromKey(targetKey) + } + b.refTypes.Store(resourceKey, types) + return nil +} + // dependencyRename returns the rename of the field a depends_on label references, nil if the label // is not a pure reference or that field is not renamed in this plan. func (b *DeploymentBundle) dependencyRename(label string) (*deployplan.ChangeDesc, error) { diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 887a046f167..27f3cd17177 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -889,6 +889,7 @@ func TestAddParentMoves(t *testing.T) { tests := []struct { name string schemaRenamed bool + volumeName string volumeChanges deployplan.Changes wantID string wantDependsOn []deployplan.DependsOnEntry @@ -910,6 +911,18 @@ func TestAddParentMoves(t *testing.T) { wantID: "main.s1-renamed.vol", wantDependsOn: []deployplan.DependsOnEntry{schema2Dep, schema1Dep}, }, + { + // The volume's own name has the schema's old value too; only schema_name followed the schema. + name: "volume named like the renamed schema", + schemaRenamed: true, + volumeName: "s1", + volumeChanges: deployplan.Changes{ + "schema_name": {Action: deployplan.Recreate, Old: "s1", New: "s2"}, + "name": {Action: deployplan.UpdateWithID, Old: "s1", New: "vol2"}, + }, + wantID: "main.s1-renamed.s1", + wantDependsOn: []deployplan.DependsOnEntry{schema2Dep, schema1Dep}, + }, { name: "schema not renamed", schemaRenamed: false, @@ -938,11 +951,16 @@ func TestAddParentMoves(t *testing.T) { Changes: tt.volumeChanges, } + volumeName := tt.volumeName + if volumeName == "" { + volumeName = "vol" + } b := &DeploymentBundle{Adapters: adapters, Plan: plan} + b.refTypes.Store(volumeKey, map[string]string{"schema_name": "schemas"}) b.StateDB.OpenWithData("state.json", dstate.Database{State: map[string]dstate.ResourceEntry{ volumeKey: { - ID: "main.s1.vol", - State: []byte(`{"catalog_name": "main", "schema_name": "s1", "name": "vol", "volume_type": "MANAGED"}`), + ID: "main.s1." + volumeName, + State: []byte(`{"catalog_name": "main", "schema_name": "s1", "name": "` + volumeName + `", "volume_type": "MANAGED"}`), DependsOn: []deployplan.DependsOnEntry{schema1Dep}, }, }}) diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index 6cfee30dc3b..84990d0883e 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -51,6 +51,10 @@ type DeploymentBundle struct { RemoteStateCache sync.Map StateCache structvar.Cache + // refTypes maps a resource key to the type of the resource each of its pure-reference + // fields references, recorded before resolveReferences drops them (see addParentMoves). + refTypes sync.Map + // DeployDurations maps a resource key to the time.Duration its create, update or // recreate took. Only successfully deployed resources are recorded; Apply runs // nodes in parallel, hence the sync.Map. From 029989b5b000d76ee2d1cf5e71ab4c487f286da0 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 12 Nov 2026 14:27:05 +0100 Subject: [PATCH 69/75] direct: recreate a model or secret where its renamed schema moved it (review C1) A schema now renames in place, and UC moves its registered models and secrets with it. They cannot follow a rename yet (registered models and UC secrets get that separately), so they are recreated; with IDFromState the recreate deletes them where they moved instead of failing on create with ALREADY_EXISTS. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 23 +++++ .../out.test.toml | 2 + .../rename_with_model_and_secret/output.txt | 95 +++++++++++++++++++ .../rename_with_model_and_secret/script | 27 ++++++ .../rename_with_model_and_secret/test.toml | 7 ++ bundle/direct/dresources/registered_model.go | 5 + bundle/direct/dresources/secret.go | 5 + 7 files changed, 164 insertions(+) create mode 100644 acceptance/bundle/resources/schemas/rename_with_model_and_secret/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/schemas/rename_with_model_and_secret/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/rename_with_model_and_secret/output.txt create mode 100644 acceptance/bundle/resources/schemas/rename_with_model_and_secret/script create mode 100644 acceptance/bundle/resources/schemas/rename_with_model_and_secret/test.toml diff --git a/acceptance/bundle/resources/schemas/rename_with_model_and_secret/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/databricks.yml.tmpl new file mode 100644 index 00000000000..b8ea3af2c01 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/databricks.yml.tmpl @@ -0,0 +1,23 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: The secret's value + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + registered_models: + model1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: mymodel + secrets: + secret1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: mysecret + value: ${var.secret_value} diff --git a/acceptance/bundle/resources/schemas/rename_with_model_and_secret/out.test.toml b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_with_model_and_secret/output.txt b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/output.txt new file mode 100644 index 00000000000..cd987eb4ca0 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/output.txt @@ -0,0 +1,95 @@ + +=== Deploy a schema with a registered model and a secret in it + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created registered_models.model1 +Created schemas.schema1 +Created secrets.secret1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema. UC moves the model and the secret with it; they cannot follow a rename yet, so they are recreated where they moved + +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> [CLI] bundle plan +recreate registered_models.model1 +update schemas.schema1 +recreate secrets.secret1 + +Plan: 2 to add, 1 to change, 2 to delete, 0 unchanged + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Recreated registered_models.model1 +Recreated secrets.secret1 +Updated schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 1 changed, 2 deleted, 0 unchanged + +>>> print_requests.py --sort //unity-catalog +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/models/main.test-schema-renamed-[UNIQUE_NAME].mymodel" +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/secrets/main.test-schema-renamed-[UNIQUE_NAME].mysecret" +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/models", + "body": { + "catalog_name": "main", + "name": "mymodel", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "main", + "name": "mysecret", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "value": "secret-value" + } +} + +>>> [CLI] registered-models get main.test-schema-renamed-[UNIQUE_NAME].mymodel +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].mymodel" +} + +>>> [CLI] secrets-uc get-secret main.test-schema-renamed-[UNIQUE_NAME].mysecret +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].mysecret" +} + +=== Redeploy is a no-op + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.registered_models.model1 + delete resources.schemas.schema1 + delete resources.secrets.secret1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/schemas/rename_with_model_and_secret/script b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/script new file mode 100644 index 00000000000..af95a0bb084 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/script @@ -0,0 +1,27 @@ +envsubst < databricks.yml.tmpl > databricks.yml +export BUNDLE_VAR_secret_value=secret-value + +NEW_SCHEMA="main.test-schema-renamed-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a registered model and a secret in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests + +title "Rename the schema. UC moves the model and the secret with it; they cannot follow a rename yet, so they are recreated where they moved" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +trace $CLI bundle plan +trace $CLI bundle deploy --auto-approve +trace print_requests.py --sort //unity-catalog +trace $CLI registered-models get "$NEW_SCHEMA.mymodel" | jq '{full_name}' +trace $CLI secrets-uc get-secret "$NEW_SCHEMA.mysecret" | jq '{full_name}' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_with_model_and_secret/test.toml b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/bundle/direct/dresources/registered_model.go b/bundle/direct/dresources/registered_model.go index 9dd97965bce..4352b848fe8 100644 --- a/bundle/direct/dresources/registered_model.go +++ b/bundle/direct/dresources/registered_model.go @@ -192,6 +192,11 @@ func (*ResourceRegisteredModel) OverrideChangeDesc(_ context.Context, path *stru return nil } +// IDFromState returns the full name, which is the ID. +func (*ResourceRegisteredModel) IDFromState(state *catalog.CreateRegisteredModelRequest) string { + return state.CatalogName + "." + state.SchemaName + "." + state.Name +} + func (r *ResourceRegisteredModel) DoDelete(ctx context.Context, id string, _ *catalog.CreateRegisteredModelRequest) error { return r.client.RegisteredModels.Delete(ctx, catalog.DeleteRegisteredModelRequest{ FullName: id, diff --git a/bundle/direct/dresources/secret.go b/bundle/direct/dresources/secret.go index 2db0a334f82..7cc30eec4d5 100644 --- a/bundle/direct/dresources/secret.go +++ b/bundle/direct/dresources/secret.go @@ -104,6 +104,11 @@ func (r *ResourceSecret) DoUpdate(ctx context.Context, id string, state *catalog return remapSecretRemote(response), nil } +// IDFromState returns the full name, which is the ID. +func (*ResourceSecret) IDFromState(state *catalog.Secret) string { + return state.CatalogName + "." + state.SchemaName + "." + state.Name +} + // DoDelete deletes the secret. func (r *ResourceSecret) DoDelete(ctx context.Context, id string, _ *catalog.Secret) error { return r.client.SecretsUc.DeleteSecret(ctx, catalog.DeleteSecretRequest{ From daba26b88be75892ad39a8ba91d9091f56cc87ac Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 12 Nov 2026 14:31:28 +0100 Subject: [PATCH 70/75] testserver: store volumes under the lowercased full name they are looked up by (review C7) Lookups were already lowercased; the stored key kept the catalog's casing, so a volume in a catalog with uppercase letters could not be found. Co-authored-by: Isaac --- libs/testserver/catalogs.go | 2 +- libs/testserver/handlers.go | 9 ++++----- libs/testserver/schemas.go | 2 +- libs/testserver/volumes.go | 10 ++++++++-- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 44c48474584..53740b3bd89 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -153,7 +153,7 @@ func (s *FakeWorkspace) moveCatalogChildren(oldCatalogName, newCatalogName strin delete(s.Volumes, key) volume.CatalogName = newCatalogName volume.FullName = newCatalogName + "." + volume.SchemaName + "." + volume.Name - s.Volumes[volume.FullName] = volume + s.Volumes[volumeKey(volume.FullName)] = volume } for key, model := range s.RegisteredModels { if model.CatalogName != oldCatalogName { diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index f840b33cf9c..c833a220a4e 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -986,11 +986,10 @@ func AddDefaultHandlers(server *Server) { return MapDelete(req.Workspace, req.Workspace.ModelProviderServices, req.Vars["name"]) }) - // Volumes: stored under their lowercased full name; UC looks a full name up case-insensitively - // (checked on AWS: a volume re-keyed as main.s.MyVolume is still read at that path). + // Volumes: looked up case-insensitively, see volumeKey. server.Handle("GET", "/api/2.1/unity-catalog/volumes/{full_name}", func(req Request) any { - return MapGetUC(req.Workspace, req.Workspace.Volumes, strings.ToLower(req.Vars["full_name"]), "Volume") + return MapGetUC(req.Workspace, req.Workspace.Volumes, volumeKey(req.Vars["full_name"]), "Volume") }) server.Handle("POST", "/api/2.1/unity-catalog/volumes", func(req Request) any { @@ -998,11 +997,11 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("PATCH", "/api/2.1/unity-catalog/volumes/{full_name}", func(req Request) any { - return req.Workspace.VolumesUpdate(req, strings.ToLower(req.Vars["full_name"])) + return req.Workspace.VolumesUpdate(req, volumeKey(req.Vars["full_name"])) }) server.Handle("DELETE", "/api/2.1/unity-catalog/volumes/{full_name}", func(req Request) any { - return MapDelete(req.Workspace, req.Workspace.Volumes, strings.ToLower(req.Vars["full_name"])) + return MapDelete(req.Workspace, req.Workspace.Volumes, volumeKey(req.Vars["full_name"])) }) // Repos: diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index 348deece67a..c934adcdc15 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -135,7 +135,7 @@ func (s *FakeWorkspace) moveSchemaChildren(catalogName, oldSchemaName, newSchema delete(s.Volumes, key) volume.SchemaName = newSchemaName volume.FullName = catalogName + "." + newSchemaName + "." + volume.Name - s.Volumes[volume.FullName] = volume + s.Volumes[volumeKey(volume.FullName)] = volume } for key, model := range s.RegisteredModels { if model.CatalogName != catalogName || model.SchemaName != oldSchemaName { diff --git a/libs/testserver/volumes.go b/libs/testserver/volumes.go index 00466f88b6f..2a23d4471e4 100644 --- a/libs/testserver/volumes.go +++ b/libs/testserver/volumes.go @@ -51,7 +51,7 @@ func (s *FakeWorkspace) VolumesCreate(req Request) Response { defer s.LockUnlock()() - s.Volumes[volume.FullName] = volume + s.Volumes[volumeKey(volume.FullName)] = volume // Register the volume path as a directory so that fs cp works without // requiring an explicit mkdir. @@ -66,6 +66,12 @@ func (s *FakeWorkspace) VolumesCreate(req Request) Response { } } +// volumeKey is the key a volume is stored under. UC looks a full name up case-insensitively +// (checked on AWS: a volume re-keyed as main.s.MyVolume is still read at that path). +func volumeKey(fullName string) string { + return strings.ToLower(fullName) +} + func (s *FakeWorkspace) VolumesUpdate(req Request, fullname string) Response { defer s.LockUnlock()() @@ -102,7 +108,7 @@ func (s *FakeWorkspace) VolumesUpdate(req Request, fullname string) Response { existing.UpdatedAt = nowMilli() existing.UpdatedBy = s.CurrentUser().UserName - s.Volumes[existing.FullName] = existing + s.Volumes[volumeKey(existing.FullName)] = existing return Response{ Body: existing, } From 032112a2b1dd202df61ad21a32497a605d678782 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 13 Nov 2026 09:55:01 +0100 Subject: [PATCH 71/75] direct: never re-key a resource's own name to follow its parent (review D3) A volume whose name references its schema's name had both fields re-keyed, so the recreate's delete went to the wrong ID. Only fields locating the parent follow it, not the resource's updatable ID field. Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 14 ++++++++++++- bundle/direct/bundle_plan_test.go | 33 ++++++++++++++++++++++++------- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 5ae6f831cca..167864fcc7b 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -818,7 +818,7 @@ func (b *DeploymentBundle) addParentMoves(plan *deployplan.Plan) error { parentType := config.GetResourceTypeFromKey(dep.Node) for pathString, ch := range entry.Changes { // Only a field referencing a resource of the parent's type followed it (schema_name for a - // schema); another one that merely had the same value, like the resource's own name, did not. + // schema); another one that merely had the same value did not. if types, ok := refTypes.(map[string]string); !ok || types[pathString] != parentType { continue } @@ -829,6 +829,10 @@ func (b *DeploymentBundle) addParentMoves(plan *deployplan.Plan) error { if err != nil { return err } + // The resource's own renamable name stays its own, even when it references the parent. + if isUpdatableIDField(adapter, path) { + continue + } if err := structaccess.Set(state, path, rename.New); err != nil { return fmt.Errorf("%s: setting %s: %w", resourceKey, pathString, err) } @@ -856,6 +860,14 @@ func (b *DeploymentBundle) addParentMoves(plan *deployplan.Plan) error { return nil } +// isUpdatableIDField reports whether path is one of the resource's updatable ID fields, its own +// renamable name. +func isUpdatableIDField(adapter *dresources.Adapter, path *structpath.PathNode) bool { + _, own := findMatchingRule(path, adapter.ResourceConfig().UpdatableIDFields) + _, generated := findMatchingRule(path, adapter.GeneratedResourceConfig().UpdatableIDFields) + return own || generated +} + // recordRefTypes records, for each pure-reference field of resourceKey, the type of the resource it // references, before resolveReferences drops the references (see addParentMoves). func (b *DeploymentBundle) recordRefTypes(resourceKey string) error { diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 27f3cd17177..a632dd11f7d 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -887,12 +887,13 @@ func TestAddParentMoves(t *testing.T) { schema2Dep := deployplan.DependsOnEntry{Node: "resources.schemas.schema2", Label: "${resources.schemas.schema2.name}"} tests := []struct { - name string - schemaRenamed bool - volumeName string - volumeChanges deployplan.Changes - wantID string - wantDependsOn []deployplan.DependsOnEntry + name string + schemaRenamed bool + volumeName string + nameRefsSchema bool + volumeChanges deployplan.Changes + wantID string + wantDependsOn []deployplan.DependsOnEntry }{ { name: "moved to another schema", @@ -923,6 +924,20 @@ func TestAddParentMoves(t *testing.T) { wantID: "main.s1-renamed.s1", wantDependsOn: []deployplan.DependsOnEntry{schema2Dep, schema1Dep}, }, + { + // The volume's own name references the schema too; it is its own name and does not follow. + name: "volume named after the renamed schema by reference", + schemaRenamed: true, + volumeName: "s1", + nameRefsSchema: true, + volumeChanges: deployplan.Changes{ + "schema_name": {Action: deployplan.UpdateWithID, Reason: deployplan.ReasonParentRenamed, Old: "s1", New: "s1-renamed"}, + "name": {Action: deployplan.UpdateWithID, Old: "s1", New: "s1-renamed"}, + "volume_type": {Action: deployplan.Recreate, Old: "MANAGED", New: "EXTERNAL"}, + }, + wantID: "main.s1-renamed.s1", + wantDependsOn: []deployplan.DependsOnEntry{schema2Dep, schema1Dep}, + }, { name: "schema not renamed", schemaRenamed: false, @@ -956,7 +971,11 @@ func TestAddParentMoves(t *testing.T) { volumeName = "vol" } b := &DeploymentBundle{Adapters: adapters, Plan: plan} - b.refTypes.Store(volumeKey, map[string]string{"schema_name": "schemas"}) + refTypes := map[string]string{"schema_name": "schemas"} + if tt.nameRefsSchema { + refTypes["name"] = "schemas" + } + b.refTypes.Store(volumeKey, refTypes) b.StateDB.OpenWithData("state.json", dstate.Database{State: map[string]dstate.ResourceEntry{ volumeKey: { ID: "main.s1." + volumeName, From 0b9652e861e4b23d25a09fa01137c3b445d0b8f5 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 13 Nov 2026 09:55:36 +0100 Subject: [PATCH 72/75] direct: a state-only update does not need DoUpdate (review D7) Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 167864fcc7b..b77535788ef 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -430,8 +430,9 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks // it can be used for variable resolution depends on several factors, see canReadRemoteCache in LookupReferencePreDeploy b.RemoteStateCache.Store(resourceKey, remoteState) - // Validate that resources without DoUpdate don't have update actions - if action == deployplan.Update && !adapter.HasDoUpdate() { + // Validate that resources without DoUpdate don't have update actions. A state-only update + // never calls it. + if action == deployplan.Update && !entry.StateOnly && !adapter.HasDoUpdate() { logdiag.LogError(ctx, fmt.Errorf("%s: resource does not support update action but plan produced update", errorPrefix)) return false } From a1a84320147cec66359d6e076367ae115c25cc0f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 13 Nov 2026 09:55:46 +0100 Subject: [PATCH 73/75] direct: log a failed optional internal-ID read at debug level (review D10) The user cannot act on it, and the next deploy records the ID anyway. Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index b77535788ef..dac21ea8c47 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -1022,7 +1022,7 @@ func ReadMissingInternalIDs(ctx context.Context, client *databricks.WorkspaceCli } if err != nil { // The ID is optional: without it a state-only update in the next deploy records it. - log.Warnf(ctx, "%s: reading id=%q to record its internal id: %s", resourceKey, record.ID, err) + log.Debugf(ctx, "%s: reading id=%q to record its internal id: %s", resourceKey, record.ID, err) continue } if err := recordInternalID(adapter, data, resourceKey, remote); err != nil { From b719f1494db292a289ac24a562a166ff094b6c24 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 13 Nov 2026 10:01:28 +0100 Subject: [PATCH 74/75] direct: share the pure-reference parsing and the parent-follow condition (review D8) Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 54 ++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index dac21ea8c47..43a58291ab2 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -267,7 +267,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks // Collected before resolveReferences, which drops the references it resolves. Only a resource // that can follow a rename needs them (see followsParentRename). var parentRenames map[string]*deployplan.ChangeDesc - if adapter.HasDoUpdateWithID() && adapter.HasIDFromState() { + if canFollowParentRename(adapter) { parentRenames, err = b.parentRenames(resourceKey) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) @@ -741,15 +741,13 @@ func (b *DeploymentBundle) parentRenames(resourceKey string) (map[string]*deploy result := make(map[string]*deployplan.ChangeDesc) for fieldPath, refString := range sv.Refs { - ref, ok := dynvar.NewRef(dyn.V(refString)) - if !ok || !ref.IsPure() { - continue - } - targetPath, err := structpath.ParsePath(ref.References()[0]) + targetKey, targetField, ok, err := pureRefTarget(refString) if err != nil { - return nil, fmt.Errorf("cannot parse reference %q: %w", refString, err) + return nil, err + } + if !ok { + continue } - targetKey, targetField := splitResourcePath(targetPath) change, err := b.renameChange(targetKey, targetField.String()) if err != nil { return nil, err @@ -878,16 +876,13 @@ func (b *DeploymentBundle) recordRefTypes(resourceKey string) error { } types := make(map[string]string, len(sv.Refs)) for fieldPath, refString := range sv.Refs { - ref, ok := dynvar.NewRef(dyn.V(refString)) - if !ok || !ref.IsPure() { - continue - } - targetPath, err := structpath.ParsePath(ref.References()[0]) + targetKey, _, ok, err := pureRefTarget(refString) if err != nil { - return fmt.Errorf("cannot parse reference %q: %w", refString, err) + return err + } + if ok { + types[fieldPath] = config.GetResourceTypeFromKey(targetKey) } - targetKey, _ := splitResourcePath(targetPath) - types[fieldPath] = config.GetResourceTypeFromKey(targetKey) } b.refTypes.Store(resourceKey, types) return nil @@ -896,16 +891,26 @@ func (b *DeploymentBundle) recordRefTypes(resourceKey string) error { // dependencyRename returns the rename of the field a depends_on label references, nil if the label // is not a pure reference or that field is not renamed in this plan. func (b *DeploymentBundle) dependencyRename(label string) (*deployplan.ChangeDesc, error) { - ref, ok := dynvar.NewRef(dyn.V(label)) + targetKey, targetField, ok, err := pureRefTarget(label) + if err != nil || !ok { + return nil, err + } + return b.renameChange(targetKey, targetField.String()) +} + +// pureRefTarget returns the resource key and field that a pure reference such as +// ${resources.schemas.s.name} points to; ok is false for anything else. +func pureRefTarget(refString string) (string, *structpath.PathNode, bool, error) { + ref, ok := dynvar.NewRef(dyn.V(refString)) if !ok || !ref.IsPure() { - return nil, nil + return "", nil, false, nil } targetPath, err := structpath.ParsePath(ref.References()[0]) if err != nil { - return nil, fmt.Errorf("cannot parse reference %q: %w", label, err) + return "", nil, false, fmt.Errorf("cannot parse reference %q: %w", refString, err) } targetKey, targetField := splitResourcePath(targetPath) - return b.renameChange(targetKey, targetField.String()) + return targetKey, targetField, true, nil } // findMoved looks for a resource missing from its recorded ID at the ID its config gives it, where a @@ -1078,7 +1083,14 @@ func hasUnresolvedIDField(adapter *dresources.Adapter, sv *structvar.StructVar) // The old values must match too: a child moved to the renamed parent from somewhere else // still has to be recreated. func followsParentRename(adapter *dresources.Adapter, parentChange, ch *deployplan.ChangeDesc) bool { - return parentChange != nil && adapter.HasDoUpdateWithID() && adapter.HasIDFromState() && structdiff.IsEqual(ch.Old, parentChange.Old) + return parentChange != nil && canFollowParentRename(adapter) && structdiff.IsEqual(ch.Old, parentChange.Old) +} + +// canFollowParentRename reports whether the resource can follow a renamed parent with an update_id: +// with no other change it is only re-keyed (StateOnly, via IDFromState), otherwise DoUpdateWithID +// addresses it at its new ID. +func canFollowParentRename(adapter *dresources.Adapter) bool { + return adapter.HasDoUpdateWithID() && adapter.HasIDFromState() } // shouldSkipNormalized skips a change that is a false diff caused by UC API From 7034294e0457676b73573e820e2b2ab6e9c05cab Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 13 Nov 2026 10:21:23 +0100 Subject: [PATCH 75/75] Make the changelog entry mirror the PR title Co-authored-by: Isaac --- .nextchanges/bundles/schema-rename.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/schema-rename.md b/.nextchanges/bundles/schema-rename.md index f728601449b..65716095967 100644 --- a/.nextchanges/bundles/schema-rename.md +++ b/.nextchanges/bundles/schema-rename.md @@ -1 +1 @@ -* Rename a schema in place when its `name` changes, instead of deleting and recreating it. Schemas and volumes inside a renamed catalog or schema now follow it to the new name, and one renamed outside the bundle is found under its new name. ([#7343](https://github.com/databricks/cli/pull/7343)) +* Rename schemas in place; schemas and volumes follow a renamed catalog or schema. ([#7343](https://github.com/databricks/cli/pull/7343))