From 0a4d9ef921a04b365c0e14e103cce874be88a866 Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Wed, 11 Nov 2026 13:07:27 +0000 Subject: [PATCH 1/3] Add isolation_mode to the catalogs bundle resource resources.Catalog embeds the SDK's CreateCatalog, which has no isolation_mode because the Create API does not accept it (UC only takes it on UpdateCatalog). The bundle schema is derived from that struct, so `bundle validate` rejected the field, and the direct engine hard-coded IsolationMode to "" in both update paths and dropped the remote value in RemapState. Add IsolationMode to resources.Catalog and introduce CatalogState (CreateCatalog plus isolation_mode) as the direct-engine state type. DoCreate issues a single-field UpdateCatalog after a successful create when the field is set; DoUpdate/DoUpdateWithID pass it through; RemapState carries the remote value so plan and drift detection see it. A backend_defaults rule makes an unset field leave the remote mode alone (UC reports OPEN for new catalogs and keeps the current mode when an update omits the field). The test server now reports OPEN on create, as UC does. Regenerates the JSON schema, enum validation, refschema output, direct engine api types and the pydabs models. Co-authored-by: Isaac --- .nextchanges/bundles/5183.md | 1 + .../configs/catalog_optional_fields.yml.tmpl | 1 + .../bundle/refschema/output/catalogs.txt | 2 +- .../catalogs/comment_out_of_band/output.txt | 5 + .../drift/managed_properties/output.txt | 5 + .../isolation_mode/databricks.yml.tmpl | 8 ++ .../catalogs/isolation_mode/out.test.toml | 2 + .../catalogs/isolation_mode/output.txt | 118 ++++++++++++++++++ .../resources/catalogs/isolation_mode/script | 35 ++++++ .../catalogs/isolation_mode/test.toml | 8 ++ bundle/config/resources/catalog.go | 4 + bundle/direct/dresources/all_test.go | 7 +- .../direct/dresources/apitypes.generated.yml | 2 +- bundle/direct/dresources/catalog.go | 72 +++++++++-- bundle/direct/dresources/catalog_test.go | 66 ++++++++++ bundle/direct/dresources/configs/catalogs.yml | 5 + bundle/internal/schema/annotations.yml | 3 + .../validation/generated/enum_fields.go | 2 + bundle/schema/jsonschema.json | 19 +++ libs/testserver/catalogs.go | 2 + libs/testserver/catalogs_test.go | 20 ++- .../databricks/bundles/catalogs/__init__.py | 6 + .../bundles/catalogs/_models/catalog.py | 14 +++ .../_models/catalog_isolation_mode.py | 12 ++ .../core/_generated/catalogs.py | 5 + 25 files changed, 404 insertions(+), 20 deletions(-) create mode 100644 .nextchanges/bundles/5183.md create mode 100644 acceptance/bundle/resources/catalogs/isolation_mode/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/catalogs/isolation_mode/out.test.toml create mode 100644 acceptance/bundle/resources/catalogs/isolation_mode/output.txt create mode 100644 acceptance/bundle/resources/catalogs/isolation_mode/script create mode 100644 acceptance/bundle/resources/catalogs/isolation_mode/test.toml create mode 100644 bundle/direct/dresources/catalog_test.go create mode 100644 python/databricks/bundles/catalogs/_models/catalog_isolation_mode.py diff --git a/.nextchanges/bundles/5183.md b/.nextchanges/bundles/5183.md new file mode 100644 index 00000000000..6e2f28306e3 --- /dev/null +++ b/.nextchanges/bundles/5183.md @@ -0,0 +1 @@ +* Add `isolation_mode` to the `catalogs` resource, so a catalog can be deployed as `OPEN` or `ISOLATED`. ([#NNNN](https://github.com/databricks/cli/pull/NNNN)) diff --git a/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl b/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl index 6a971ec93d2..3c9b17f4104 100644 --- a/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl +++ b/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl @@ -7,6 +7,7 @@ resources: name: test-catalog-$UNIQUE_NAME comment: This is a test catalog custom_max_retention_hours: 48 # CAN_REMOVE + isolation_mode: ISOLATED # CAN_REMOVE managed_encryption_settings: customer_managed_key_id: 00000000-0000-0000-0000-000000000000 properties: diff --git a/acceptance/bundle/refschema/output/catalogs.txt b/acceptance/bundle/refschema/output/catalogs.txt index feaee00d2ca..42fb0f78ead 100644 --- a/acceptance/bundle/refschema/output/catalogs.txt +++ b/acceptance/bundle/refschema/output/catalogs.txt @@ -12,7 +12,7 @@ resources.catalogs.*.effective_predictive_optimization_flag.value catalog.Enable resources.catalogs.*.enable_predictive_optimization catalog.EnablePredictiveOptimization REMOTE resources.catalogs.*.full_name string REMOTE resources.catalogs.*.id string INPUT -resources.catalogs.*.isolation_mode catalog.CatalogIsolationMode REMOTE +resources.catalogs.*.isolation_mode catalog.CatalogIsolationMode ALL resources.catalogs.*.lifecycle resources.Lifecycle INPUT resources.catalogs.*.lifecycle.depends_on []resources.DependsOn INPUT resources.catalogs.*.lifecycle.depends_on[*] resources.DependsOn INPUT diff --git a/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt index 58bebd5aac2..d6307dcaf45 100644 --- a/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt @@ -13,6 +13,11 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "comment": { "action": "update", "remote": "set outside the bundle" + }, + "isolation_mode": { + "action": "skip", + "reason": "backend_default", + "remote": "OPEN" } } diff --git a/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt b/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt index 5c22507bb39..5ed71f54cc9 100644 --- a/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt +++ b/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt @@ -13,6 +13,11 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The remote-only properties map is skipped as a backend default (confirms the matched rule) >>> [CLI] bundle plan --output json { + "isolation_mode": { + "action": "skip", + "reason": "backend_default", + "remote": "OPEN" + }, "properties": { "action": "skip", "reason": "backend_default", diff --git a/acceptance/bundle/resources/catalogs/isolation_mode/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/isolation_mode/databricks.yml.tmpl new file mode 100644 index 00000000000..3b3fceb61dc --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation_mode/databricks.yml.tmpl @@ -0,0 +1,8 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + catalog1: + name: test-catalog-$UNIQUE_NAME + isolation_mode: ISOLATED diff --git a/acceptance/bundle/resources/catalogs/isolation_mode/out.test.toml b/acceptance/bundle/resources/catalogs/isolation_mode/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation_mode/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/isolation_mode/output.txt b/acceptance/bundle/resources/catalogs/isolation_mode/output.txt new file mode 100644 index 00000000000..ece71c172b9 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation_mode/output.txt @@ -0,0 +1,118 @@ + +=== Validate accepts isolation_mode +>>> [CLI] bundle validate --output json +{ + "name": "test-catalog-[UNIQUE_NAME]", + "isolation_mode": "ISOLATED" +} + +=== Deploy: CreateCatalog does not accept isolation_mode, so the catalog is created and then patched +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog/catalogs +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/catalogs", + "body": { + "name": "test-catalog-[UNIQUE_NAME]" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test-catalog-[UNIQUE_NAME]", + "body": { + "isolation_mode": "ISOLATED" + } +} + +>>> [CLI] catalogs get test-catalog-[UNIQUE_NAME] +{ + "name": "test-catalog-[UNIQUE_NAME]", + "isolation_mode": "ISOLATED" +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Removing isolation_mode from the config leaves the remote value alone +>>> update_file.py databricks.yml isolation_mode: ISOLATED # isolation_mode removed + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated catalogs.catalog1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog/catalogs +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test-catalog-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0 + } +} + +>>> [CLI] catalogs get test-catalog-[UNIQUE_NAME] +{ + "name": "test-catalog-[UNIQUE_NAME]", + "isolation_mode": "ISOLATED" +} + +>>> [CLI] bundle plan --output json +{ + "comment": { + "action": "skip", + "reason": "empty", + "remote": "" + }, + "custom_max_retention_hours": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "isolation_mode": { + "action": "skip", + "reason": "backend_default", + "remote": "ISOLATED" + } +} + +=== Setting OPEN patches the catalog +>>> update_file.py databricks.yml # isolation_mode removed isolation_mode: OPEN + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated catalogs.catalog1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog/catalogs +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test-catalog-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "isolation_mode": "OPEN" + } +} + +>>> [CLI] catalogs get test-catalog-[UNIQUE_NAME] +{ + "name": "test-catalog-[UNIQUE_NAME]", + "isolation_mode": "OPEN" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.catalog1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/catalogs/isolation_mode/script b/acceptance/bundle/resources/catalogs/isolation_mode/script new file mode 100644 index 00000000000..27ab364c730 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation_mode/script @@ -0,0 +1,35 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +CATALOG="test-catalog-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog/catalogs &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Validate accepts isolation_mode" +trace $CLI bundle validate --output json | jq '.resources.catalogs.catalog1 | {name, isolation_mode}' + +title "Deploy: CreateCatalog does not accept isolation_mode, so the catalog is created and then patched" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog/catalogs +trace $CLI catalogs get "$CATALOG" | jq '{name, isolation_mode}' + +title "Redeploy is a no-op" +trace $CLI bundle plan + +title "Removing isolation_mode from the config leaves the remote value alone" +trace update_file.py databricks.yml "isolation_mode: ISOLATED" "# isolation_mode removed" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog/catalogs +trace $CLI catalogs get "$CATALOG" | jq '{name, isolation_mode}' +trace $CLI bundle plan --output json | nostamp | jq '.plan[].changes' + +title "Setting OPEN patches the catalog" +trace update_file.py databricks.yml "# isolation_mode removed" "isolation_mode: OPEN" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog/catalogs +trace $CLI catalogs get "$CATALOG" | jq '{name, isolation_mode}' diff --git a/acceptance/bundle/resources/catalogs/isolation_mode/test.toml b/acceptance/bundle/resources/catalogs/isolation_mode/test.toml new file mode 100644 index 00000000000..d7d2f90fc9e --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation_mode/test.toml @@ -0,0 +1,8 @@ +# Local only: the create-then-update sequence is direct-engine-specific, and whether an +# ISOLATED catalog stays readable from the deploying workspace without an explicit +# workspace binding has not been verified against a real workspace. + +Ignore = [ + ".databricks", + "databricks.yml", +] diff --git a/bundle/config/resources/catalog.go b/bundle/config/resources/catalog.go index 3fc4a5c89dc..94181ea6bd7 100644 --- a/bundle/config/resources/catalog.go +++ b/bundle/config/resources/catalog.go @@ -18,6 +18,10 @@ type Catalog struct { ID string `json:"id,omitempty" bundle:"readonly"` catalog.CreateCatalog + // IsolationMode is declared here rather than taken from CreateCatalog because the + // Create API does not accept it; UC only takes it on update. + IsolationMode catalog.CatalogIsolationMode `json:"isolation_mode,omitempty"` + // List of grants to apply on this catalog. Grants []catalog.PrivilegeAssignment `json:"grants,omitempty"` } diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index b542481be83..2501abcce4f 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -47,8 +47,11 @@ var testConfig map[string]any = map[string]any{ "catalogs": &resources.Catalog{ Name: "mycatalog", Comment: "Test catalog", - // Note: EnablePredictiveOptimization and IsolationMode cannot be set during creation, - // only during updates. They are not included in the test config. + // Note: EnablePredictiveOptimization cannot be set during creation, + // only during updates. It is not included in the test config. + // IsolationMode has the same restriction but is applied by DoCreate with a + // follow-up update, so it is set here to cover that path. + IsolationMode: catalog.CatalogIsolationModeIsolated, }, "external_locations": &resources.ExternalLocation{ diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index e5ee771955e..28d43c57b4f 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -4,7 +4,7 @@ alerts: sql.AlertV2 apps: apps.App -catalogs: catalog.CreateCatalog +catalogs: catalog.CatalogInfo cluster_policies: compute.CreatePolicy diff --git a/bundle/direct/dresources/catalog.go b/bundle/direct/dresources/catalog.go index 638138bfbda..53d9b3b23f1 100644 --- a/bundle/direct/dresources/catalog.go +++ b/bundle/direct/dresources/catalog.go @@ -2,14 +2,33 @@ package dresources import ( "context" + "fmt" "slices" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) +// CatalogState is CreateCatalog plus isolation_mode, which the Create API does not +// accept; DoCreate applies it with a follow-up update. +type CatalogState struct { + catalog.CreateCatalog + IsolationMode catalog.CatalogIsolationMode `json:"isolation_mode,omitempty"` +} + +// Custom marshalers needed because embedded catalog.CreateCatalog has its own MarshalJSON +// which would otherwise take over and ignore the additional fields. +func (s *CatalogState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s CatalogState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + type ResourceCatalog struct { client *databricks.WorkspaceClient } @@ -18,12 +37,15 @@ func (*ResourceCatalog) New(client *databricks.WorkspaceClient) *ResourceCatalog return &ResourceCatalog{client: client} } -func (*ResourceCatalog) PrepareState(input *resources.Catalog) *catalog.CreateCatalog { - return &input.CreateCatalog +func (*ResourceCatalog) PrepareState(input *resources.Catalog) *CatalogState { + return &CatalogState{ + CreateCatalog: input.CreateCatalog, + IsolationMode: input.IsolationMode, + } } -func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *catalog.CreateCatalog { - return &catalog.CreateCatalog{ +func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *CatalogState { + return &CatalogState{ Comment: info.Comment, ConnectionName: info.ConnectionName, CustomMaxRetentionHours: info.CustomMaxRetentionHours, @@ -35,6 +57,7 @@ func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *catalog.CreateCat ShareName: info.ShareName, StorageRoot: info.StorageRoot, ForceSendFields: utils.FilterFields[catalog.CreateCatalog](info.ForceSendFields), + IsolationMode: info.IsolationMode, } } @@ -42,11 +65,34 @@ func (r *ResourceCatalog) DoRead(ctx context.Context, id string) (*catalog.Catal return r.client.Catalogs.GetByName(ctx, id) } -func (r *ResourceCatalog) DoCreate(ctx context.Context, config *catalog.CreateCatalog) (string, *catalog.CatalogInfo, error) { - response, err := r.client.Catalogs.Create(ctx, *config) +func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (string, *catalog.CatalogInfo, error) { + response, err := r.client.Catalogs.Create(ctx, config.CreateCatalog) if err != nil || response == nil { return "", nil, err } + if config.IsolationMode == "" { + return response.Name, response, nil + } + + // UC only accepts isolation_mode on update, so a configured value takes a second call. + response, err = r.client.Catalogs.Update(ctx, catalog.UpdateCatalog{ + Comment: "", + CustomMaxRetentionHours: 0, + EnablePredictiveOptimization: "", + IsolationMode: config.IsolationMode, + ManagedEncryptionSettings: nil, + Name: response.Name, + NewName: "", + Options: nil, + Owner: "", + Properties: nil, + ForceSendFields: nil, + }) + if err != nil { + // The catalog exists but is not in state, so the next deploy fails with "already + // exists"; name the cause here since the caller only knows a create failed. + return "", nil, fmt.Errorf("catalog %s was created but setting isolation_mode failed: %w", config.Name, err) + } return response.Name, response, nil } @@ -59,19 +105,19 @@ func (r *ResourceCatalog) DoCreate(ctx context.Context, config *catalog.CreateCa var catalogForceSend = []string{"Comment", "CustomMaxRetentionHours"} // DoUpdate updates the catalog in place and returns remote state. -func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *catalog.CreateCatalog, _ *PlanEntry) (*catalog.CatalogInfo, error) { +func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (*catalog.CatalogInfo, error) { updateRequest := catalog.UpdateCatalog{ Comment: config.Comment, CustomMaxRetentionHours: config.CustomMaxRetentionHours, EnablePredictiveOptimization: "", // Not supported by DABs - IsolationMode: "", // Not supported by DABs + IsolationMode: config.IsolationMode, ManagedEncryptionSettings: config.ManagedEncryptionSettings, Name: id, NewName: "", // Only set if name actually changes (see DoUpdateWithID) Options: config.Options, Owner: "", // Not supported by DABs Properties: config.Properties, - ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](append(slices.Clone(catalogForceSend), config.ForceSendFields...), "EnablePredictiveOptimization", "IsolationMode", "Owner"), + ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](append(slices.Clone(catalogForceSend), config.ForceSendFields...), "EnablePredictiveOptimization", "Owner"), } response, err := r.client.Catalogs.Update(ctx, updateRequest) @@ -83,19 +129,19 @@ func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *catal } // DoUpdateWithID updates the catalog and returns the new ID if the name changes. -func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateCatalog, _ *PlanEntry) (string, *catalog.CatalogInfo, error) { +func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (string, *catalog.CatalogInfo, error) { updateRequest := catalog.UpdateCatalog{ Comment: config.Comment, CustomMaxRetentionHours: config.CustomMaxRetentionHours, EnablePredictiveOptimization: "", // Not supported by DABs - IsolationMode: "", // Not supported by DABs + IsolationMode: config.IsolationMode, ManagedEncryptionSettings: config.ManagedEncryptionSettings, Name: id, NewName: "", // Initialized below if needed Options: config.Options, Owner: "", // Not supported by DABs Properties: config.Properties, - ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](append(slices.Clone(catalogForceSend), config.ForceSendFields...), "EnablePredictiveOptimization", "IsolationMode", "Owner"), + ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](append(slices.Clone(catalogForceSend), config.ForceSendFields...), "EnablePredictiveOptimization", "Owner"), } if config.Name != id { @@ -116,7 +162,7 @@ func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config return newID, response, nil } -func (r *ResourceCatalog) DoDelete(ctx context.Context, id string, _ *catalog.CreateCatalog) error { +func (r *ResourceCatalog) DoDelete(ctx context.Context, id string, _ *CatalogState) error { return r.client.Catalogs.Delete(ctx, catalog.DeleteCatalogRequest{ Name: id, Force: true, diff --git a/bundle/direct/dresources/catalog_test.go b/bundle/direct/dresources/catalog_test.go new file mode 100644 index 00000000000..8d9596ea233 --- /dev/null +++ b/bundle/direct/dresources/catalog_test.go @@ -0,0 +1,66 @@ +package dresources + +import ( + "testing" + + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResourceCatalog_DoCreate_IsolationMode(t *testing.T) { + tests := []struct { + name string + isolationMode catalog.CatalogIsolationMode + // The fake, like UC, reports OPEN for a catalog created without isolation_mode. + want catalog.CatalogIsolationMode + }{ + {name: "unset keeps the UC default", isolationMode: "", want: catalog.CatalogIsolationModeOpen}, + {name: "ISOLATED is applied after create", isolationMode: catalog.CatalogIsolationModeIsolated, want: catalog.CatalogIsolationModeIsolated}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, client := setupTestServerClient(t) + adapter := (*ResourceCatalog)(nil).New(client) + ctx := t.Context() + + id, remote, err := adapter.DoCreate(ctx, &CatalogState{ + Name: "test_catalog", + IsolationMode: tt.isolationMode, + }) + require.NoError(t, err) + assert.Equal(t, "test_catalog", id) + assert.Equal(t, tt.want, remote.IsolationMode) + + read, err := adapter.DoRead(ctx, id) + require.NoError(t, err) + assert.Equal(t, tt.want, read.IsolationMode) + }) + } +} + +func TestResourceCatalog_DoUpdate_IsolationMode(t *testing.T) { + _, client := setupTestServerClient(t) + adapter := (*ResourceCatalog)(nil).New(client) + ctx := t.Context() + + config := &CatalogState{ + Name: "test_catalog", + IsolationMode: catalog.CatalogIsolationModeIsolated, + } + id, _, err := adapter.DoCreate(ctx, config) + require.NoError(t, err) + + // An update that omits isolation_mode leaves the remote value alone (partial-update + // semantics), which is what lets a config drop the field without forcing OPEN. + config.IsolationMode = "" + remote, err := adapter.DoUpdate(ctx, id, config, &PlanEntry{}) + require.NoError(t, err) + assert.Equal(t, catalog.CatalogIsolationModeIsolated, remote.IsolationMode) + + config.IsolationMode = catalog.CatalogIsolationModeOpen + remote, err = adapter.DoUpdate(ctx, id, config, &PlanEntry{}) + require.NoError(t, err) + assert.Equal(t, catalog.CatalogIsolationModeOpen, remote.IsolationMode) +} diff --git a/bundle/direct/dresources/configs/catalogs.yml b/bundle/direct/dresources/configs/catalogs.yml index 143a74c447d..c1e1893af42 100644 --- a/bundle/direct/dresources/configs/catalogs.yml +++ b/bundle/direct/dresources/configs/catalogs.yml @@ -19,3 +19,8 @@ updatable_id_fields: backend_defaults: # UC auto-populates unity.catalog.managed..defaults.* keys after create. - field: properties['unity.catalog.managed.*.defaults.*'] + # UC reports OPEN for a catalog created without isolation_mode and keeps the current + # mode when an update omits it. A config without the field therefore leaves the remote + # value alone (OPEN by default, or ISOLATED when set outside the bundle) rather than + # forcing it back to OPEN. + - field: isolation_mode diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 1744a23441d..5f23eb94d15 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -544,6 +544,9 @@ resources: privileges: - ALL_PRIVILEGES ``` + "isolation_mode": + "description": |- + Whether the catalog is accessible from all workspaces (`OPEN`) or only from the workspaces it is bound to (`ISOLATED`). Workspace bindings are not managed by the bundle. When omitted, the catalog keeps its current isolation mode. "lifecycle": "description": |- Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index c075aacd577..c2b7a092791 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -40,6 +40,8 @@ var EnumFields = map[string][]string{ "resources.apps.*.resources[*].uc_securable.permission": {"EXECUTE", "MODIFY", "READ_VOLUME", "SELECT", "USE_CONNECTION", "WRITE_VOLUME"}, "resources.apps.*.resources[*].uc_securable.securable_type": {"CONNECTION", "FUNCTION", "MODEL_SERVICE", "TABLE", "VOLUME"}, + "resources.catalogs.*.isolation_mode": {"ISOLATED", "OPEN"}, + "resources.cluster_policies.*.permissions[*].level": {"CAN_USE"}, "resources.clusters.*.aws_attributes.availability": {"ON_DEMAND", "SPOT", "SPOT_WITH_FALLBACK"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 86ae495dd1d..9671cb7219f 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -375,6 +375,10 @@ "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/catalog.PrivilegeAssignment", "markdownDescription": "A Sequence of Unity Catalog privilege grants on this securable, where each item grants a set of `privileges` to a `principal` (a user, group, or service principal).\n\nSee [link](https://docs.databricks.com/data-governance/unity-catalog/manage-privileges/index.html)." }, + "isolation_mode": { + "description": "Whether the catalog is accessible from all workspaces (`OPEN`) or only from the workspaces it is bound to (`ISOLATED`). Workspace bindings are not managed by the bundle. When omitted, the catalog keeps its current isolation mode.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.CatalogIsolationMode" + }, "lifecycle": { "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" @@ -5761,6 +5765,21 @@ } ] }, + "catalog.CatalogIsolationMode": { + "oneOf": [ + { + "type": "string", + "enum": [ + "OPEN", + "ISOLATED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "catalog.EncryptionDetails": { "oneOf": [ { diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 6e9ed132551..44c48474584 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -68,6 +68,8 @@ func (s *FakeWorkspace) CatalogsCreate(req Request) Response { MetastoreId: nextUUID(), Owner: s.CurrentUser().UserName, CatalogType: catalog.CatalogTypeManagedCatalog, + // UC defaults a new catalog to OPEN; isolation_mode is only settable on update. + IsolationMode: catalog.CatalogIsolationModeOpen, } catalogInfo.UpdatedAt = catalogInfo.CreatedAt if catalogInfo.Properties == nil && createRequest.Name == catalogNameManagedDefaults { diff --git a/libs/testserver/catalogs_test.go b/libs/testserver/catalogs_test.go index aa98c477479..de924c08060 100644 --- a/libs/testserver/catalogs_test.go +++ b/libs/testserver/catalogs_test.go @@ -107,16 +107,29 @@ func TestCatalogsCreate_RequestCoversEveryAcceptedField(t *testing.T) { } } +// UC reports OPEN for a new catalog; isolation_mode is only settable on update. +func TestCatalogsCreate_DefaultsIsolationModeToOpen(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + + response := workspace.CatalogsCreate(Request{Body: []byte(createCatalogRequest)}) + require.Equal(t, 0, response.StatusCode) + + info, ok := response.Body.(catalog.CatalogInfo) + require.True(t, ok) + assert.Equal(t, catalog.CatalogIsolationModeOpen, info.IsolationMode) +} + func TestCatalogsUpdate_AppliesUpdatableFields(t *testing.T) { workspace := NewFakeWorkspace("http://test", "dbapi123") require.Equal(t, 0, workspace.CatalogsCreate(Request{Body: []byte(createCatalogRequest)}).StatusCode) - // UpdateCatalog also accepts enable_predictive_optimization and isolation_mode, - // which DABs deliberately leaves unset (see dresources.ResourceCatalog.DoUpdate), - // so the fake does not model them. + // UpdateCatalog also accepts enable_predictive_optimization, which DABs + // deliberately leaves unset (see dresources.ResourceCatalog.DoUpdate), so the + // fake does not model it. response := workspace.CatalogsUpdate(Request{Body: []byte(`{ "comment": "updated", "custom_max_retention_hours": 72, + "isolation_mode": "ISOLATED", "managed_encryption_settings": {"customer_managed_key_id": "key-2"}, "options": {"opt": "3"}, "properties": {"prop": "4"} @@ -128,6 +141,7 @@ func TestCatalogsUpdate_AppliesUpdatableFields(t *testing.T) { assert.Equal(t, "updated", info.Comment) assert.Equal(t, int64(72), info.CustomMaxRetentionHours) + assert.Equal(t, catalog.CatalogIsolationModeIsolated, info.IsolationMode) require.NotNil(t, info.ManagedEncryptionSettings) assert.Equal(t, "key-2", info.ManagedEncryptionSettings.CustomerManagedKeyId) assert.Equal(t, map[string]string{"opt": "3"}, info.Options) diff --git a/python/databricks/bundles/catalogs/__init__.py b/python/databricks/bundles/catalogs/__init__.py index fced08f1bb2..8aef38b14cc 100644 --- a/python/databricks/bundles/catalogs/__init__.py +++ b/python/databricks/bundles/catalogs/__init__.py @@ -11,6 +11,8 @@ "CatalogGrantParam", "CatalogGrantPrivilege", "CatalogGrantPrivilegeParam", + "CatalogIsolationMode", + "CatalogIsolationModeParam", "CatalogParam", "DependsOn", "DependsOnDict", @@ -39,6 +41,10 @@ CatalogDict, CatalogParam, ) +from databricks.bundles.catalogs._models.catalog_isolation_mode import ( + CatalogIsolationMode, + CatalogIsolationModeParam, +) from databricks.bundles.catalogs._models.depends_on import ( DependsOn, DependsOnDict, diff --git a/python/databricks/bundles/catalogs/_models/catalog.py b/python/databricks/bundles/catalogs/_models/catalog.py index bdb397b0cb0..81720474a9e 100644 --- a/python/databricks/bundles/catalogs/_models/catalog.py +++ b/python/databricks/bundles/catalogs/_models/catalog.py @@ -3,6 +3,10 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, TypedDict +from databricks.bundles.catalogs._models.catalog_isolation_mode import ( + CatalogIsolationMode, + CatalogIsolationModeParam, +) from databricks.bundles.catalogs._models.encryption_settings import ( EncryptionSettings, EncryptionSettingsParam, @@ -55,6 +59,11 @@ class Catalog(Resource): The Unity Catalog privileges to grant to principals on this securable. """ + isolation_mode: VariableOrOptional[CatalogIsolationMode] = None + """ + Whether the catalog is accessible from all workspaces (`OPEN`) or only from the workspaces it is bound to (`ISOLATED`). Workspace bindings are not managed by the bundle. When omitted, the catalog keeps its current isolation mode. + """ + lifecycle: VariableOrOptional[Lifecycle] = None """ Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. @@ -128,6 +137,11 @@ class CatalogDict(TypedDict, total=False): The Unity Catalog privileges to grant to principals on this securable. """ + isolation_mode: VariableOrOptional[CatalogIsolationModeParam] + """ + Whether the catalog is accessible from all workspaces (`OPEN`) or only from the workspaces it is bound to (`ISOLATED`). Workspace bindings are not managed by the bundle. When omitted, the catalog keeps its current isolation mode. + """ + lifecycle: VariableOrOptional[LifecycleParam] """ Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. diff --git a/python/databricks/bundles/catalogs/_models/catalog_isolation_mode.py b/python/databricks/bundles/catalogs/_models/catalog_isolation_mode.py new file mode 100644 index 00000000000..bc6f3f5d15b --- /dev/null +++ b/python/databricks/bundles/catalogs/_models/catalog_isolation_mode.py @@ -0,0 +1,12 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from enum import Enum +from typing import Literal + + +class CatalogIsolationMode(Enum): + OPEN = "OPEN" + ISOLATED = "ISOLATED" + + +CatalogIsolationModeParam = Literal["OPEN", "ISOLATED"] | CatalogIsolationMode diff --git a/python/databricks_tests/core/_generated/catalogs.py b/python/databricks_tests/core/_generated/catalogs.py index 117d964071c..575e3934359 100644 --- a/python/databricks_tests/core/_generated/catalogs.py +++ b/python/databricks_tests/core/_generated/catalogs.py @@ -1,6 +1,9 @@ # Code generated by pydabs-codegen. DO NOT EDIT. from databricks.bundles.catalogs._models.catalog import Catalog +from databricks.bundles.catalogs._models.catalog_isolation_mode import ( + CatalogIsolationMode, +) from databricks.bundles.catalogs._models.encryption_settings import EncryptionSettings from databricks.bundles.catalogs._models.lifecycle import Lifecycle from databricks.bundles.catalogs._models.privilege_assignment import PrivilegeAssignment @@ -15,6 +18,7 @@ def _test_case(): add_resource=Resources.add_catalog, dict_example={ "grants": [{}], + "isolation_mode": "OPEN", "lifecycle": {}, "managed_encryption_settings": {}, "name": "name", @@ -23,6 +27,7 @@ def _test_case(): }, dataclass_example=Catalog( grants=[PrivilegeAssignment()], + isolation_mode=CatalogIsolationMode.OPEN, lifecycle=Lifecycle(), managed_encryption_settings=EncryptionSettings(), name="name", From b961dabf4715ad0e3671568e844b77d54ab6fd94 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 11 Nov 2026 14:28:12 +0100 Subject: [PATCH 2/3] Link changelog fragment to PR #7463 --- .nextchanges/bundles/5183.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/5183.md b/.nextchanges/bundles/5183.md index 6e2f28306e3..df12f2581c2 100644 --- a/.nextchanges/bundles/5183.md +++ b/.nextchanges/bundles/5183.md @@ -1 +1 @@ -* Add `isolation_mode` to the `catalogs` resource, so a catalog can be deployed as `OPEN` or `ISOLATED`. ([#NNNN](https://github.com/databricks/cli/pull/NNNN)) +* Add `isolation_mode` to the `catalogs` resource, so a catalog can be deployed as `OPEN` or `ISOLATED`. ([#7463](https://github.com/databricks/cli/pull/7463)) From 57a818027a6011e617183edbc4fba0f3714aad25 Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Wed, 11 Nov 2026 18:56:58 +0000 Subject: [PATCH 3/3] Regenerate catalogs goldens for the title newline change Main commit 8b4f052d5 (#7003) gave the acceptance `title` helper a trailing newline, so every `=== Section` header is now followed by a blank line. The isolation_mode golden was generated before that change and went stale when main was merged into this branch. Co-authored-by: Isaac --- .../bundle/resources/catalogs/isolation_mode/output.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/acceptance/bundle/resources/catalogs/isolation_mode/output.txt b/acceptance/bundle/resources/catalogs/isolation_mode/output.txt index ece71c172b9..c37befe061b 100644 --- a/acceptance/bundle/resources/catalogs/isolation_mode/output.txt +++ b/acceptance/bundle/resources/catalogs/isolation_mode/output.txt @@ -1,5 +1,6 @@ === Validate accepts isolation_mode + >>> [CLI] bundle validate --output json { "name": "test-catalog-[UNIQUE_NAME]", @@ -7,6 +8,7 @@ } === Deploy: CreateCatalog does not accept isolation_mode, so the catalog is created and then patched + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created catalogs.catalog1 @@ -36,10 +38,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Redeploy is a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Removing isolation_mode from the config leaves the remote value alone + >>> update_file.py databricks.yml isolation_mode: ISOLATED # isolation_mode removed >>> [CLI] bundle deploy @@ -84,6 +88,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Setting OPEN patches the catalog + >>> update_file.py databricks.yml # isolation_mode removed isolation_mode: OPEN >>> [CLI] bundle deploy