diff --git a/.nextchanges/bundles/5183.md b/.nextchanges/bundles/5183.md new file mode 100644 index 00000000000..df12f2581c2 --- /dev/null +++ b/.nextchanges/bundles/5183.md @@ -0,0 +1 @@ +* Add `isolation_mode` to the `catalogs` resource, so a catalog can be deployed as `OPEN` or `ISOLATED`. ([#7463](https://github.com/databricks/cli/pull/7463)) diff --git a/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl b/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl index 6a971ec93d2..3c9b17f4104 100644 --- a/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl +++ b/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl @@ -7,6 +7,7 @@ resources: name: test-catalog-$UNIQUE_NAME comment: This is a test catalog custom_max_retention_hours: 48 # CAN_REMOVE + isolation_mode: ISOLATED # CAN_REMOVE managed_encryption_settings: customer_managed_key_id: 00000000-0000-0000-0000-000000000000 properties: diff --git a/acceptance/bundle/refschema/output/catalogs.txt b/acceptance/bundle/refschema/output/catalogs.txt index feaee00d2ca..42fb0f78ead 100644 --- a/acceptance/bundle/refschema/output/catalogs.txt +++ b/acceptance/bundle/refschema/output/catalogs.txt @@ -12,7 +12,7 @@ resources.catalogs.*.effective_predictive_optimization_flag.value catalog.Enable resources.catalogs.*.enable_predictive_optimization catalog.EnablePredictiveOptimization REMOTE resources.catalogs.*.full_name string REMOTE resources.catalogs.*.id string INPUT -resources.catalogs.*.isolation_mode catalog.CatalogIsolationMode REMOTE +resources.catalogs.*.isolation_mode catalog.CatalogIsolationMode ALL resources.catalogs.*.lifecycle resources.Lifecycle INPUT resources.catalogs.*.lifecycle.depends_on []resources.DependsOn INPUT resources.catalogs.*.lifecycle.depends_on[*] resources.DependsOn INPUT diff --git a/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt index 65bbc43d3df..5da6c37fe2a 100644 --- a/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt @@ -16,6 +16,11 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "comment": { "action": "update", "remote": "set outside the bundle" + }, + "isolation_mode": { + "action": "skip", + "reason": "backend_default", + "remote": "OPEN" } } diff --git a/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt b/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt index 2f01b81cc37..886246cec10 100644 --- a/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt +++ b/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt @@ -16,6 +16,11 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> [CLI] bundle plan --output json { + "isolation_mode": { + "action": "skip", + "reason": "backend_default", + "remote": "OPEN" + }, "properties": { "action": "skip", "reason": "backend_default", diff --git a/acceptance/bundle/resources/catalogs/isolation_mode/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/isolation_mode/databricks.yml.tmpl new file mode 100644 index 00000000000..3b3fceb61dc --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation_mode/databricks.yml.tmpl @@ -0,0 +1,8 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + catalog1: + name: test-catalog-$UNIQUE_NAME + isolation_mode: ISOLATED diff --git a/acceptance/bundle/resources/catalogs/isolation_mode/out.test.toml b/acceptance/bundle/resources/catalogs/isolation_mode/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation_mode/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/isolation_mode/output.txt b/acceptance/bundle/resources/catalogs/isolation_mode/output.txt new file mode 100644 index 00000000000..c37befe061b --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation_mode/output.txt @@ -0,0 +1,123 @@ + +=== Validate accepts isolation_mode + +>>> [CLI] bundle validate --output json +{ + "name": "test-catalog-[UNIQUE_NAME]", + "isolation_mode": "ISOLATED" +} + +=== Deploy: CreateCatalog does not accept isolation_mode, so the catalog is created and then patched + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog/catalogs +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/catalogs", + "body": { + "name": "test-catalog-[UNIQUE_NAME]" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test-catalog-[UNIQUE_NAME]", + "body": { + "isolation_mode": "ISOLATED" + } +} + +>>> [CLI] catalogs get test-catalog-[UNIQUE_NAME] +{ + "name": "test-catalog-[UNIQUE_NAME]", + "isolation_mode": "ISOLATED" +} + +=== Redeploy is a no-op + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Removing isolation_mode from the config leaves the remote value alone + +>>> update_file.py databricks.yml isolation_mode: ISOLATED # isolation_mode removed + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated catalogs.catalog1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog/catalogs +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test-catalog-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0 + } +} + +>>> [CLI] catalogs get test-catalog-[UNIQUE_NAME] +{ + "name": "test-catalog-[UNIQUE_NAME]", + "isolation_mode": "ISOLATED" +} + +>>> [CLI] bundle plan --output json +{ + "comment": { + "action": "skip", + "reason": "empty", + "remote": "" + }, + "custom_max_retention_hours": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "isolation_mode": { + "action": "skip", + "reason": "backend_default", + "remote": "ISOLATED" + } +} + +=== Setting OPEN patches the catalog + +>>> update_file.py databricks.yml # isolation_mode removed isolation_mode: OPEN + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated catalogs.catalog1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog/catalogs +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test-catalog-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "isolation_mode": "OPEN" + } +} + +>>> [CLI] catalogs get test-catalog-[UNIQUE_NAME] +{ + "name": "test-catalog-[UNIQUE_NAME]", + "isolation_mode": "OPEN" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.catalog1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/catalogs/isolation_mode/script b/acceptance/bundle/resources/catalogs/isolation_mode/script new file mode 100644 index 00000000000..27ab364c730 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation_mode/script @@ -0,0 +1,35 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +CATALOG="test-catalog-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog/catalogs &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Validate accepts isolation_mode" +trace $CLI bundle validate --output json | jq '.resources.catalogs.catalog1 | {name, isolation_mode}' + +title "Deploy: CreateCatalog does not accept isolation_mode, so the catalog is created and then patched" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog/catalogs +trace $CLI catalogs get "$CATALOG" | jq '{name, isolation_mode}' + +title "Redeploy is a no-op" +trace $CLI bundle plan + +title "Removing isolation_mode from the config leaves the remote value alone" +trace update_file.py databricks.yml "isolation_mode: ISOLATED" "# isolation_mode removed" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog/catalogs +trace $CLI catalogs get "$CATALOG" | jq '{name, isolation_mode}' +trace $CLI bundle plan --output json | nostamp | jq '.plan[].changes' + +title "Setting OPEN patches the catalog" +trace update_file.py databricks.yml "# isolation_mode removed" "isolation_mode: OPEN" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog/catalogs +trace $CLI catalogs get "$CATALOG" | jq '{name, isolation_mode}' diff --git a/acceptance/bundle/resources/catalogs/isolation_mode/test.toml b/acceptance/bundle/resources/catalogs/isolation_mode/test.toml new file mode 100644 index 00000000000..d7d2f90fc9e --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation_mode/test.toml @@ -0,0 +1,8 @@ +# Local only: the create-then-update sequence is direct-engine-specific, and whether an +# ISOLATED catalog stays readable from the deploying workspace without an explicit +# workspace binding has not been verified against a real workspace. + +Ignore = [ + ".databricks", + "databricks.yml", +] diff --git a/bundle/config/resources/catalog.go b/bundle/config/resources/catalog.go index 3fc4a5c89dc..94181ea6bd7 100644 --- a/bundle/config/resources/catalog.go +++ b/bundle/config/resources/catalog.go @@ -18,6 +18,10 @@ type Catalog struct { ID string `json:"id,omitempty" bundle:"readonly"` catalog.CreateCatalog + // IsolationMode is declared here rather than taken from CreateCatalog because the + // Create API does not accept it; UC only takes it on update. + IsolationMode catalog.CatalogIsolationMode `json:"isolation_mode,omitempty"` + // List of grants to apply on this catalog. Grants []catalog.PrivilegeAssignment `json:"grants,omitempty"` } diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 605f530a94c..b835dd78948 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -47,8 +47,11 @@ var testConfig map[string]any = map[string]any{ "catalogs": &resources.Catalog{ Name: "mycatalog", Comment: "Test catalog", - // Note: EnablePredictiveOptimization and IsolationMode cannot be set during creation, - // only during updates. They are not included in the test config. + // Note: EnablePredictiveOptimization cannot be set during creation, + // only during updates. It is not included in the test config. + // IsolationMode has the same restriction but is applied by DoCreate with a + // follow-up update, so it is set here to cover that path. + IsolationMode: catalog.CatalogIsolationModeIsolated, }, "external_locations": &resources.ExternalLocation{ diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index 798c78f950c..db57a08bc8d 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -4,7 +4,7 @@ alerts: sql.AlertV2 apps: apps.App -catalogs: catalog.CreateCatalog +catalogs: catalog.CatalogInfo cluster_policies: compute.CreatePolicy diff --git a/bundle/direct/dresources/catalog.go b/bundle/direct/dresources/catalog.go index 638138bfbda..53d9b3b23f1 100644 --- a/bundle/direct/dresources/catalog.go +++ b/bundle/direct/dresources/catalog.go @@ -2,14 +2,33 @@ package dresources import ( "context" + "fmt" "slices" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) +// CatalogState is CreateCatalog plus isolation_mode, which the Create API does not +// accept; DoCreate applies it with a follow-up update. +type CatalogState struct { + catalog.CreateCatalog + IsolationMode catalog.CatalogIsolationMode `json:"isolation_mode,omitempty"` +} + +// Custom marshalers needed because embedded catalog.CreateCatalog has its own MarshalJSON +// which would otherwise take over and ignore the additional fields. +func (s *CatalogState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s CatalogState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + type ResourceCatalog struct { client *databricks.WorkspaceClient } @@ -18,12 +37,15 @@ func (*ResourceCatalog) New(client *databricks.WorkspaceClient) *ResourceCatalog return &ResourceCatalog{client: client} } -func (*ResourceCatalog) PrepareState(input *resources.Catalog) *catalog.CreateCatalog { - return &input.CreateCatalog +func (*ResourceCatalog) PrepareState(input *resources.Catalog) *CatalogState { + return &CatalogState{ + CreateCatalog: input.CreateCatalog, + IsolationMode: input.IsolationMode, + } } -func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *catalog.CreateCatalog { - return &catalog.CreateCatalog{ +func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *CatalogState { + return &CatalogState{ Comment: info.Comment, ConnectionName: info.ConnectionName, CustomMaxRetentionHours: info.CustomMaxRetentionHours, @@ -35,6 +57,7 @@ func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *catalog.CreateCat ShareName: info.ShareName, StorageRoot: info.StorageRoot, ForceSendFields: utils.FilterFields[catalog.CreateCatalog](info.ForceSendFields), + IsolationMode: info.IsolationMode, } } @@ -42,11 +65,34 @@ func (r *ResourceCatalog) DoRead(ctx context.Context, id string) (*catalog.Catal return r.client.Catalogs.GetByName(ctx, id) } -func (r *ResourceCatalog) DoCreate(ctx context.Context, config *catalog.CreateCatalog) (string, *catalog.CatalogInfo, error) { - response, err := r.client.Catalogs.Create(ctx, *config) +func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (string, *catalog.CatalogInfo, error) { + response, err := r.client.Catalogs.Create(ctx, config.CreateCatalog) if err != nil || response == nil { return "", nil, err } + if config.IsolationMode == "" { + return response.Name, response, nil + } + + // UC only accepts isolation_mode on update, so a configured value takes a second call. + response, err = r.client.Catalogs.Update(ctx, catalog.UpdateCatalog{ + Comment: "", + CustomMaxRetentionHours: 0, + EnablePredictiveOptimization: "", + IsolationMode: config.IsolationMode, + ManagedEncryptionSettings: nil, + Name: response.Name, + NewName: "", + Options: nil, + Owner: "", + Properties: nil, + ForceSendFields: nil, + }) + if err != nil { + // The catalog exists but is not in state, so the next deploy fails with "already + // exists"; name the cause here since the caller only knows a create failed. + return "", nil, fmt.Errorf("catalog %s was created but setting isolation_mode failed: %w", config.Name, err) + } return response.Name, response, nil } @@ -59,19 +105,19 @@ func (r *ResourceCatalog) DoCreate(ctx context.Context, config *catalog.CreateCa var catalogForceSend = []string{"Comment", "CustomMaxRetentionHours"} // DoUpdate updates the catalog in place and returns remote state. -func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *catalog.CreateCatalog, _ *PlanEntry) (*catalog.CatalogInfo, error) { +func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (*catalog.CatalogInfo, error) { updateRequest := catalog.UpdateCatalog{ Comment: config.Comment, CustomMaxRetentionHours: config.CustomMaxRetentionHours, EnablePredictiveOptimization: "", // Not supported by DABs - IsolationMode: "", // Not supported by DABs + IsolationMode: config.IsolationMode, ManagedEncryptionSettings: config.ManagedEncryptionSettings, Name: id, NewName: "", // Only set if name actually changes (see DoUpdateWithID) Options: config.Options, Owner: "", // Not supported by DABs Properties: config.Properties, - ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](append(slices.Clone(catalogForceSend), config.ForceSendFields...), "EnablePredictiveOptimization", "IsolationMode", "Owner"), + ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](append(slices.Clone(catalogForceSend), config.ForceSendFields...), "EnablePredictiveOptimization", "Owner"), } response, err := r.client.Catalogs.Update(ctx, updateRequest) @@ -83,19 +129,19 @@ func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *catal } // DoUpdateWithID updates the catalog and returns the new ID if the name changes. -func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateCatalog, _ *PlanEntry) (string, *catalog.CatalogInfo, error) { +func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (string, *catalog.CatalogInfo, error) { updateRequest := catalog.UpdateCatalog{ Comment: config.Comment, CustomMaxRetentionHours: config.CustomMaxRetentionHours, EnablePredictiveOptimization: "", // Not supported by DABs - IsolationMode: "", // Not supported by DABs + IsolationMode: config.IsolationMode, ManagedEncryptionSettings: config.ManagedEncryptionSettings, Name: id, NewName: "", // Initialized below if needed Options: config.Options, Owner: "", // Not supported by DABs Properties: config.Properties, - ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](append(slices.Clone(catalogForceSend), config.ForceSendFields...), "EnablePredictiveOptimization", "IsolationMode", "Owner"), + ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](append(slices.Clone(catalogForceSend), config.ForceSendFields...), "EnablePredictiveOptimization", "Owner"), } if config.Name != id { @@ -116,7 +162,7 @@ func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config return newID, response, nil } -func (r *ResourceCatalog) DoDelete(ctx context.Context, id string, _ *catalog.CreateCatalog) error { +func (r *ResourceCatalog) DoDelete(ctx context.Context, id string, _ *CatalogState) error { return r.client.Catalogs.Delete(ctx, catalog.DeleteCatalogRequest{ Name: id, Force: true, diff --git a/bundle/direct/dresources/catalog_test.go b/bundle/direct/dresources/catalog_test.go new file mode 100644 index 00000000000..8d9596ea233 --- /dev/null +++ b/bundle/direct/dresources/catalog_test.go @@ -0,0 +1,66 @@ +package dresources + +import ( + "testing" + + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResourceCatalog_DoCreate_IsolationMode(t *testing.T) { + tests := []struct { + name string + isolationMode catalog.CatalogIsolationMode + // The fake, like UC, reports OPEN for a catalog created without isolation_mode. + want catalog.CatalogIsolationMode + }{ + {name: "unset keeps the UC default", isolationMode: "", want: catalog.CatalogIsolationModeOpen}, + {name: "ISOLATED is applied after create", isolationMode: catalog.CatalogIsolationModeIsolated, want: catalog.CatalogIsolationModeIsolated}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, client := setupTestServerClient(t) + adapter := (*ResourceCatalog)(nil).New(client) + ctx := t.Context() + + id, remote, err := adapter.DoCreate(ctx, &CatalogState{ + Name: "test_catalog", + IsolationMode: tt.isolationMode, + }) + require.NoError(t, err) + assert.Equal(t, "test_catalog", id) + assert.Equal(t, tt.want, remote.IsolationMode) + + read, err := adapter.DoRead(ctx, id) + require.NoError(t, err) + assert.Equal(t, tt.want, read.IsolationMode) + }) + } +} + +func TestResourceCatalog_DoUpdate_IsolationMode(t *testing.T) { + _, client := setupTestServerClient(t) + adapter := (*ResourceCatalog)(nil).New(client) + ctx := t.Context() + + config := &CatalogState{ + Name: "test_catalog", + IsolationMode: catalog.CatalogIsolationModeIsolated, + } + id, _, err := adapter.DoCreate(ctx, config) + require.NoError(t, err) + + // An update that omits isolation_mode leaves the remote value alone (partial-update + // semantics), which is what lets a config drop the field without forcing OPEN. + config.IsolationMode = "" + remote, err := adapter.DoUpdate(ctx, id, config, &PlanEntry{}) + require.NoError(t, err) + assert.Equal(t, catalog.CatalogIsolationModeIsolated, remote.IsolationMode) + + config.IsolationMode = catalog.CatalogIsolationModeOpen + remote, err = adapter.DoUpdate(ctx, id, config, &PlanEntry{}) + require.NoError(t, err) + assert.Equal(t, catalog.CatalogIsolationModeOpen, remote.IsolationMode) +} diff --git a/bundle/direct/dresources/configs/catalogs.yml b/bundle/direct/dresources/configs/catalogs.yml index 143a74c447d..c1e1893af42 100644 --- a/bundle/direct/dresources/configs/catalogs.yml +++ b/bundle/direct/dresources/configs/catalogs.yml @@ -19,3 +19,8 @@ updatable_id_fields: backend_defaults: # UC auto-populates unity.catalog.managed..defaults.* keys after create. - field: properties['unity.catalog.managed.*.defaults.*'] + # UC reports OPEN for a catalog created without isolation_mode and keeps the current + # mode when an update omits it. A config without the field therefore leaves the remote + # value alone (OPEN by default, or ISOLATED when set outside the bundle) rather than + # forcing it back to OPEN. + - field: isolation_mode diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 9dff59d5399..230a7459dd7 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -544,6 +544,9 @@ resources: privileges: - ALL_PRIVILEGES ``` + "isolation_mode": + "description": |- + Whether the catalog is accessible from all workspaces (`OPEN`) or only from the workspaces it is bound to (`ISOLATED`). Workspace bindings are not managed by the bundle. When omitted, the catalog keeps its current isolation mode. "lifecycle": "description": |- Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index c075aacd577..c2b7a092791 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -40,6 +40,8 @@ var EnumFields = map[string][]string{ "resources.apps.*.resources[*].uc_securable.permission": {"EXECUTE", "MODIFY", "READ_VOLUME", "SELECT", "USE_CONNECTION", "WRITE_VOLUME"}, "resources.apps.*.resources[*].uc_securable.securable_type": {"CONNECTION", "FUNCTION", "MODEL_SERVICE", "TABLE", "VOLUME"}, + "resources.catalogs.*.isolation_mode": {"ISOLATED", "OPEN"}, + "resources.cluster_policies.*.permissions[*].level": {"CAN_USE"}, "resources.clusters.*.aws_attributes.availability": {"ON_DEMAND", "SPOT", "SPOT_WITH_FALLBACK"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 255ac4a3746..afb5a80db72 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -375,6 +375,10 @@ "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/catalog.PrivilegeAssignment", "markdownDescription": "A Sequence of Unity Catalog privilege grants on this securable, where each item grants a set of `privileges` to a `principal` (a user, group, or service principal).\n\nSee [link](https://docs.databricks.com/data-governance/unity-catalog/manage-privileges/index.html)." }, + "isolation_mode": { + "description": "Whether the catalog is accessible from all workspaces (`OPEN`) or only from the workspaces it is bound to (`ISOLATED`). Workspace bindings are not managed by the bundle. When omitted, the catalog keeps its current isolation mode.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.CatalogIsolationMode" + }, "lifecycle": { "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" @@ -5817,6 +5821,21 @@ } ] }, + "catalog.CatalogIsolationMode": { + "oneOf": [ + { + "type": "string", + "enum": [ + "OPEN", + "ISOLATED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "catalog.EncryptionDetails": { "oneOf": [ { diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 6e9ed132551..44c48474584 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -68,6 +68,8 @@ func (s *FakeWorkspace) CatalogsCreate(req Request) Response { MetastoreId: nextUUID(), Owner: s.CurrentUser().UserName, CatalogType: catalog.CatalogTypeManagedCatalog, + // UC defaults a new catalog to OPEN; isolation_mode is only settable on update. + IsolationMode: catalog.CatalogIsolationModeOpen, } catalogInfo.UpdatedAt = catalogInfo.CreatedAt if catalogInfo.Properties == nil && createRequest.Name == catalogNameManagedDefaults { diff --git a/libs/testserver/catalogs_test.go b/libs/testserver/catalogs_test.go index aa98c477479..de924c08060 100644 --- a/libs/testserver/catalogs_test.go +++ b/libs/testserver/catalogs_test.go @@ -107,16 +107,29 @@ func TestCatalogsCreate_RequestCoversEveryAcceptedField(t *testing.T) { } } +// UC reports OPEN for a new catalog; isolation_mode is only settable on update. +func TestCatalogsCreate_DefaultsIsolationModeToOpen(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + + response := workspace.CatalogsCreate(Request{Body: []byte(createCatalogRequest)}) + require.Equal(t, 0, response.StatusCode) + + info, ok := response.Body.(catalog.CatalogInfo) + require.True(t, ok) + assert.Equal(t, catalog.CatalogIsolationModeOpen, info.IsolationMode) +} + func TestCatalogsUpdate_AppliesUpdatableFields(t *testing.T) { workspace := NewFakeWorkspace("http://test", "dbapi123") require.Equal(t, 0, workspace.CatalogsCreate(Request{Body: []byte(createCatalogRequest)}).StatusCode) - // UpdateCatalog also accepts enable_predictive_optimization and isolation_mode, - // which DABs deliberately leaves unset (see dresources.ResourceCatalog.DoUpdate), - // so the fake does not model them. + // UpdateCatalog also accepts enable_predictive_optimization, which DABs + // deliberately leaves unset (see dresources.ResourceCatalog.DoUpdate), so the + // fake does not model it. response := workspace.CatalogsUpdate(Request{Body: []byte(`{ "comment": "updated", "custom_max_retention_hours": 72, + "isolation_mode": "ISOLATED", "managed_encryption_settings": {"customer_managed_key_id": "key-2"}, "options": {"opt": "3"}, "properties": {"prop": "4"} @@ -128,6 +141,7 @@ func TestCatalogsUpdate_AppliesUpdatableFields(t *testing.T) { assert.Equal(t, "updated", info.Comment) assert.Equal(t, int64(72), info.CustomMaxRetentionHours) + assert.Equal(t, catalog.CatalogIsolationModeIsolated, info.IsolationMode) require.NotNil(t, info.ManagedEncryptionSettings) assert.Equal(t, "key-2", info.ManagedEncryptionSettings.CustomerManagedKeyId) assert.Equal(t, map[string]string{"opt": "3"}, info.Options) diff --git a/python/databricks/bundles/catalogs/__init__.py b/python/databricks/bundles/catalogs/__init__.py index fced08f1bb2..8aef38b14cc 100644 --- a/python/databricks/bundles/catalogs/__init__.py +++ b/python/databricks/bundles/catalogs/__init__.py @@ -11,6 +11,8 @@ "CatalogGrantParam", "CatalogGrantPrivilege", "CatalogGrantPrivilegeParam", + "CatalogIsolationMode", + "CatalogIsolationModeParam", "CatalogParam", "DependsOn", "DependsOnDict", @@ -39,6 +41,10 @@ CatalogDict, CatalogParam, ) +from databricks.bundles.catalogs._models.catalog_isolation_mode import ( + CatalogIsolationMode, + CatalogIsolationModeParam, +) from databricks.bundles.catalogs._models.depends_on import ( DependsOn, DependsOnDict, diff --git a/python/databricks/bundles/catalogs/_models/catalog.py b/python/databricks/bundles/catalogs/_models/catalog.py index bdb397b0cb0..81720474a9e 100644 --- a/python/databricks/bundles/catalogs/_models/catalog.py +++ b/python/databricks/bundles/catalogs/_models/catalog.py @@ -3,6 +3,10 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, TypedDict +from databricks.bundles.catalogs._models.catalog_isolation_mode import ( + CatalogIsolationMode, + CatalogIsolationModeParam, +) from databricks.bundles.catalogs._models.encryption_settings import ( EncryptionSettings, EncryptionSettingsParam, @@ -55,6 +59,11 @@ class Catalog(Resource): The Unity Catalog privileges to grant to principals on this securable. """ + isolation_mode: VariableOrOptional[CatalogIsolationMode] = None + """ + Whether the catalog is accessible from all workspaces (`OPEN`) or only from the workspaces it is bound to (`ISOLATED`). Workspace bindings are not managed by the bundle. When omitted, the catalog keeps its current isolation mode. + """ + lifecycle: VariableOrOptional[Lifecycle] = None """ Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. @@ -128,6 +137,11 @@ class CatalogDict(TypedDict, total=False): The Unity Catalog privileges to grant to principals on this securable. """ + isolation_mode: VariableOrOptional[CatalogIsolationModeParam] + """ + Whether the catalog is accessible from all workspaces (`OPEN`) or only from the workspaces it is bound to (`ISOLATED`). Workspace bindings are not managed by the bundle. When omitted, the catalog keeps its current isolation mode. + """ + lifecycle: VariableOrOptional[LifecycleParam] """ Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. diff --git a/python/databricks/bundles/catalogs/_models/catalog_isolation_mode.py b/python/databricks/bundles/catalogs/_models/catalog_isolation_mode.py new file mode 100644 index 00000000000..bc6f3f5d15b --- /dev/null +++ b/python/databricks/bundles/catalogs/_models/catalog_isolation_mode.py @@ -0,0 +1,12 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from enum import Enum +from typing import Literal + + +class CatalogIsolationMode(Enum): + OPEN = "OPEN" + ISOLATED = "ISOLATED" + + +CatalogIsolationModeParam = Literal["OPEN", "ISOLATED"] | CatalogIsolationMode diff --git a/python/databricks_tests/core/_generated/catalogs.py b/python/databricks_tests/core/_generated/catalogs.py index 117d964071c..575e3934359 100644 --- a/python/databricks_tests/core/_generated/catalogs.py +++ b/python/databricks_tests/core/_generated/catalogs.py @@ -1,6 +1,9 @@ # Code generated by pydabs-codegen. DO NOT EDIT. from databricks.bundles.catalogs._models.catalog import Catalog +from databricks.bundles.catalogs._models.catalog_isolation_mode import ( + CatalogIsolationMode, +) from databricks.bundles.catalogs._models.encryption_settings import EncryptionSettings from databricks.bundles.catalogs._models.lifecycle import Lifecycle from databricks.bundles.catalogs._models.privilege_assignment import PrivilegeAssignment @@ -15,6 +18,7 @@ def _test_case(): add_resource=Resources.add_catalog, dict_example={ "grants": [{}], + "isolation_mode": "OPEN", "lifecycle": {}, "managed_encryption_settings": {}, "name": "name", @@ -23,6 +27,7 @@ def _test_case(): }, dataclass_example=Catalog( grants=[PrivilegeAssignment()], + isolation_mode=CatalogIsolationMode.OPEN, lifecycle=Lifecycle(), managed_encryption_settings=EncryptionSettings(), name="name",