From 0725cb9e34b648b167c5302abfffdab570156487 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 12 Nov 2026 10:28:31 +0000 Subject: [PATCH 1/4] Detect remote permission changes on unmanaged resources in config-remote-sync --- .../config-remote-sync/job_fields/script | 15 --- .../config-remote-sync/pipeline_fields/script | 15 --- .../unmanaged_permissions/databricks.yml.tmpl | 25 +++++ .../unmanaged_permissions/out.test.toml | 3 + .../unmanaged_permissions/output.txt | 59 ++++++++++ .../unmanaged_permissions/script | 35 ++++++ .../unmanaged_permissions/test.toml | 4 + bundle/configsync/diff.go | 52 ++++++++- bundle/configsync/diff_test.go | 55 +++++++++ bundle/direct/bind.go | 4 +- bundle/direct/bundle_plan.go | 104 +++++++++++++++--- bundle/direct/bundle_plan_test.go | 28 +++++ bundle/direct/dresources/permissions.go | 7 ++ bundle/phases/deploy.go | 2 +- bundle/phases/destroy.go | 2 +- bundle/statemgmt/direct_migration.go | 2 +- cmd/bundle/config_remote_sync.go | 2 +- 17 files changed, 359 insertions(+), 55 deletions(-) create mode 100644 acceptance/bundle/config-remote-sync/unmanaged_permissions/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/unmanaged_permissions/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/unmanaged_permissions/output.txt create mode 100644 acceptance/bundle/config-remote-sync/unmanaged_permissions/script create mode 100644 acceptance/bundle/config-remote-sync/unmanaged_permissions/test.toml diff --git a/acceptance/bundle/config-remote-sync/job_fields/script b/acceptance/bundle/config-remote-sync/job_fields/script index 27b7c8a3abc..e854123cfb9 100755 --- a/acceptance/bundle/config-remote-sync/job_fields/script +++ b/acceptance/bundle/config-remote-sync/job_fields/script @@ -40,21 +40,6 @@ if "tags" not in r: r["tags"]["team"] = "data" EOF -# TODO add support for permissions -# title "Modify job permissions remotely" -# current_user=$($CLI current-user me --output json | jq -r .userName) -# cat > permissions.json < permissions.json <>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -14,4 +14,9 @@ + num_workers: 1 + ++ permissions: ++ - level: CAN_MANAGE ++ user_name: viewer@example.com ++ - level: CAN_MANAGE_RUN ++ user_name: admin@example.com + pipelines: + my_pipeline: +@@ -20,4 +25,7 @@ + - notebook: + path: /Users/{{workspace_user_name}}/notebook ++ permissions: ++ - group_name: users ++ level: CAN_VIEW + + targets: + +=== Sync again: the permissions are now in the config +No changes detected. + + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + delete resources.pipelines.my_pipeline + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.my_pipeline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/config-remote-sync/unmanaged_permissions/script b/acceptance/bundle/config-remote-sync/unmanaged_permissions/script new file mode 100644 index 00000000000..c69f5fe5d23 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/unmanaged_permissions/script @@ -0,0 +1,35 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +$CLI bundle deploy +job_id="$(read_id.py my_job)" +pipeline_id="$(read_id.py my_pipeline)" + +title "Add permissions remotely to resources that declare none" +# The owner entry is managed by deploy, so it must not be written to the config. +$CLI permissions set jobs $job_id --json '{"access_control_list":[ + {"permission_level":"IS_OWNER","user_name":"'"$CURRENT_USER_NAME"'"}, + {"permission_level":"CAN_MANAGE","user_name":"viewer@example.com"}, + {"permission_level":"CAN_MANAGE_RUN","user_name":"admin@example.com"} +]}' > /dev/null +$CLI permissions set pipelines $pipeline_id --json '{"access_control_list":[ + {"permission_level":"IS_OWNER","user_name":"'"$CURRENT_USER_NAME"'"}, + {"permission_level":"CAN_VIEW","group_name":"users"} +]}' > /dev/null + +title "Detect and save changes" +cp databricks.yml databricks.yml.backup +$CLI bundle config-remote-sync --save + +title "Configuration changes" +trace diff.py databricks.yml.backup databricks.yml +rm databricks.yml.backup + +title "Sync again: the permissions are now in the config" +$CLI bundle config-remote-sync diff --git a/acceptance/bundle/config-remote-sync/unmanaged_permissions/test.toml b/acceptance/bundle/config-remote-sync/unmanaged_permissions/test.toml new file mode 100644 index 00000000000..a96be3a4c44 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/unmanaged_permissions/test.toml @@ -0,0 +1,4 @@ +RecordRequests = false +Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"] + +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true" diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index e42f0a1d83a..6567bf8ae3d 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -8,10 +8,12 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structdiff" + "github.com/databricks/databricks-sdk-go/service/iam" ) type OperationType string @@ -148,7 +150,8 @@ func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan // in bundle/direct/bundle_plan.go). config-remote-sync cannot write them back // to YAML: their fields (e.g. object_id) are server-populated with no source // location, and bundle-level permissions have no per-resource YAML node at all, - // so resolving them would fail the whole sync. Skip them instead. + // so resolving them would fail the whole sync. Skip them instead; only the + // permissions of resources that declare none are added below, as a whole. if isPermissionsOrGrantsSubResource(resourceKey) { continue } @@ -187,5 +190,52 @@ func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan log.Debugf(ctx, "Resource %s has %d changes", resourceKey, len(resourceChanges)) } + for resourceKey, entry := range plan.Plan { + permissions := unmanagedPermissions(b, resourceKey, entry) + if len(permissions) == 0 { + continue + } + value, err := normalizeValue(permissions) + if err != nil { + return nil, fmt.Errorf("failed to compute config change for %s: %w", resourceKey, err) + } + parentKey := strings.TrimSuffix(resourceKey, ".permissions") + if changes[parentKey] == nil { + changes[parentKey] = make(ResourceChanges) + } + changes[parentKey]["permissions"] = &ConfigChangeDesc{ + Operation: OperationAdd, + Value: value, + } + } + return changes, nil } + +// unmanagedPermissions returns the remote permissions the plan read for a resource whose config +// declares none (see CalculatePlan), without the owner and the current user's CAN_MANAGE: +// FixPermissions adds those on deploy, so they do not belong in the config. +func unmanagedPermissions(b *bundle.Bundle, resourceKey string, entry *deployplan.PlanEntry) []dresources.StatePermission { + remote, ok := entry.RemoteState.(*dresources.PermissionsState) + if !ok { + return nil + } + path, err := dyn.NewPathFromString(resourceKey) + if err != nil { + return nil + } + if _, err := dyn.GetByPath(b.Config.Value(), path); err == nil { + return nil + } + + currentUser := b.Config.Workspace.CurrentUser.UserName + var result []dresources.StatePermission + for _, p := range remote.EmbeddedSlice { + isCurrentUser := p.UserName == currentUser || p.ServicePrincipalName == currentUser + if p.Level == iam.PermissionLevelIsOwner || (p.Level == iam.PermissionLevelCanManage && isCurrentUser) { + continue + } + result = append(result, p) + } + return result +} diff --git a/bundle/configsync/diff_test.go b/bundle/configsync/diff_test.go index b16885f6b71..181aaa4e19c 100644 --- a/bundle/configsync/diff_test.go +++ b/bundle/configsync/diff_test.go @@ -4,7 +4,12 @@ import ( "testing" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/jsonloader" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -298,3 +303,53 @@ func TestExtractChangesSkipsPermissionsAndGrants(t *testing.T) { require.True(t, hasNamedPerms, "a resource named permissions must not be skipped") assert.Contains(t, namedPerms, "description") } + +func TestExtractChangesAddsUnmanagedPermissions(t *testing.T) { + value, err := jsonloader.LoadJSON([]byte(`{"resources": {"jobs": { + "unmanaged": {"name": "unmanaged"}, + "managed": {"name": "managed", "permissions": [{"level": "CAN_VIEW", "group_name": "users"}]} + }}}`), "test config") + require.NoError(t, err) + b := &bundle.Bundle{} + require.NoError(t, b.Config.Mutate(func(dyn.Value) (dyn.Value, error) { return value, nil })) + b.Config.Workspace.CurrentUser = &config.User{User: &iam.User{UserName: "me@example.com"}} + + remote := func(permissions ...dresources.StatePermission) *dresources.PermissionsState { + return &dresources.PermissionsState{ObjectID: "/jobs/123", EmbeddedSlice: permissions} + } + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.unmanaged": {}, + "resources.jobs.unmanaged.permissions": { + Action: deployplan.Skip, + RemoteState: remote( + dresources.StatePermission{Level: iam.PermissionLevelIsOwner, UserName: "me@example.com"}, + dresources.StatePermission{Level: iam.PermissionLevelCanManage, UserName: "me@example.com"}, + dresources.StatePermission{Level: iam.PermissionLevelCanView, GroupName: "users"}, + dresources.StatePermission{Level: iam.PermissionLevelCanManage, UserName: "other@example.com"}, + ), + }, + "resources.jobs.owner_only.permissions": { + Action: deployplan.Skip, + RemoteState: remote(dresources.StatePermission{Level: iam.PermissionLevelIsOwner, UserName: "me@example.com"}), + }, + "resources.jobs.managed": {}, + "resources.jobs.managed.permissions": { + Action: deployplan.Skip, + RemoteState: remote(dresources.StatePermission{Level: iam.PermissionLevelCanManageRun, GroupName: "admins"}), + }, + }} + + changes, err := ExtractChanges(t.Context(), b, plan) + require.NoError(t, err) + assert.Equal(t, Changes{ + "resources.jobs.unmanaged": { + "permissions": { + Operation: OperationAdd, + Value: []any{ + map[string]any{"level": "CAN_VIEW", "group_name": "users"}, + map[string]any{"level": "CAN_MANAGE", "user_name": "other@example.com"}, + }, + }, + }, + }, changes) +} diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 7fbf7ede662..ce7306d856f 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -124,7 +124,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac os.Remove(tmpStatePath) return nil, err } - plan, err := b.CalculatePlan(ctx, client, configRoot) + plan, err := b.CalculatePlan(ctx, client, configRoot, false) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -197,7 +197,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac os.Remove(tmpStatePath) return nil, err } - plan, err = b.CalculatePlan(ctx, client, configRoot) + plan, err = b.CalculatePlan(ctx, client, configRoot, false) if _, ferr := b.StateDB.Finalize(ctx); ferr != nil { log.Warnf(ctx, "failed to finalize state: %v", ferr) } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 237fcc75521..cc5cb13afbb 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -161,7 +161,11 @@ func (b *DeploymentBundle) StampDeploymentIdForFirstVersion(deploymentID string) // CalculatePlan computes the deployment plan by comparing local config against remote state. // StateDB must already be open for read before calling this function. -func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root) (*deployplan.Plan, error) { +// +// With readUnmanagedPermissions, deployed resources whose config and state have no permissions +// also get a Skip ".permissions" entry holding the remote permissions in RemoteState. Only +// config-remote-sync sets it, to write remote permission changes back to the config. +func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, readUnmanagedPermissions bool) (*deployplan.Plan, error) { b.StateDB.AssertOpenedForRead() err := b.init(client) @@ -169,7 +173,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return nil, err } - plan, err := b.makePlan(ctx, configRoot, &b.StateDB.Data) + plan, unmanagedPermissions, err := b.makePlan(ctx, configRoot, &b.StateDB.Data, readUnmanagedPermissions) if err != nil { return nil, fmt.Errorf("reading config: %w", err) } @@ -269,6 +273,11 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } + if unmanagedPermissions[resourceKey] { + b.readUnmanagedPermissions(ctx, adapter, resourceKey, entry) + return true + } + dbentry, hasEntry := b.StateDB.GetResourceEntry(resourceKey) // Tolerate empty-ID entries from older partial-recreate failures // (apply.Recreate now deletes state on the way through, but pre-fix @@ -415,6 +424,29 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return plan, nil } +// readUnmanagedPermissions records the remote permissions of a resource whose config declares +// none. There is nothing to deploy for it, so a failed read is not fatal: the user may not be +// allowed to read the permissions of a resource they can deploy. +func (b *DeploymentBundle) readUnmanagedPermissions(ctx context.Context, adapter *dresources.Adapter, resourceKey string, entry *deployplan.PlanEntry) { + entry.Action = deployplan.Skip + + sv, _ := b.StateCache.Load(resourceKey) + if len(sv.Refs) > 0 { + // The parent's ID is only known after deploy, e.g. it is recreated. + return + } + + id := sv.Value.(*dresources.PermissionsState).ObjectID + remoteState, err := retry.With(ctx, retry.IsTransientRead, func() (any, error) { + return adapter.DoRead(ctx, id) + }) + if err != nil { + log.Warnf(ctx, "reading permissions of %s id=%q: %s", resourceKey, id, err) + return + } + entry.RemoteState = remoteState +} + func getMaxAction(m map[string]*deployplan.ChangeDesc) deployplan.ActionType { result := deployplan.Skip for _, ch := range m { @@ -1149,7 +1181,7 @@ func (b *DeploymentBundle) resolveReferences(ctx context.Context, resourceKey st return true } -func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root, db *dstate.Database) (*deployplan.Plan, error) { +func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root, db *dstate.Database, readUnmanagedPermissions bool) (*deployplan.Plan, map[string]bool, error) { p := deployplan.NewPlanDirect() // Copy state metadata to plan for validation during apply @@ -1189,11 +1221,17 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root }, ) if err != nil { - return nil, err + return nil, nil, err } } } + var unmanagedPermissions map[string]bool + if readUnmanagedPermissions { + unmanagedPermissions = unmanagedPermissionsNodes(nodes, db) + nodes = slices.AppendSeq(nodes, maps.Keys(unmanagedPermissions)) + } + slices.Sort(nodes) for _, node := range nodes { delete(existingKeys, node) @@ -1201,22 +1239,25 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root prefix := "cannot plan " + node inputConfig, err := configRoot.GetResourceConfig(node) if err != nil { - return nil, err + return nil, nil, err + } + if unmanagedPermissions[node] { + inputConfig = reflect.New(config.ResourcesTypes()[config.GetResourceTypeFromKey(node)]).Interface() } adapter, err := b.getAdapterForKey(node) if err != nil { - return nil, fmt.Errorf("%s: %w", prefix, err) + return nil, nil, fmt.Errorf("%s: %w", prefix, err) } inputStructVar, err := adapter.PrepareInputConfig(inputConfig, node) if err != nil { - return nil, fmt.Errorf("%s: %w", prefix, err) + return nil, nil, fmt.Errorf("%s: %w", prefix, err) } newStateConfig, err := adapter.PrepareState(inputStructVar.Value) if err != nil { - return nil, fmt.Errorf("%s: %w", prefix, err) + return nil, nil, fmt.Errorf("%s: %w", prefix, err) } // Unescape "$${...}" to a literal "${...}" in the typed state, which is what @@ -1226,7 +1267,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root // extractReferences below still sees the escaped form and does not mistake // these placeholders for bundle references. if err := unescapeRefs(newStateConfig); err != nil { - return nil, fmt.Errorf("%s: %w", prefix, err) + return nil, nil, fmt.Errorf("%s: %w", prefix, err) } // New nodes only: a node with state must stay in the plan, otherwise emptying it plans nothing. @@ -1234,7 +1275,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root if _, hasState := db.State[node]; !hasState { empty, err := adapter.IsEmptyState(newStateConfig) if err != nil { - return nil, fmt.Errorf("%s: %w", prefix, err) + return nil, nil, fmt.Errorf("%s: %w", prefix, err) } if empty { continue @@ -1245,9 +1286,12 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root // This means input and state must be compatible: input can have more fields, but existing fields should not be moved // This means one cannot refer to fields not present in state (e.g. ${resources.jobs.foo.permissions}) - refs, err := extractReferences(configRoot.Value(), node, adapter.StateType()) - if err != nil { - return nil, fmt.Errorf("failed to read references from config for %s: %w", node, err) + refs := make(map[string]string) + if !unmanagedPermissions[node] { + refs, err = extractReferences(configRoot.Value(), node, adapter.StateType()) + if err != nil { + return nil, nil, fmt.Errorf("failed to read references from config for %s: %w", node, err) + } } maps.Copy(refs, inputStructVar.Refs) @@ -1262,7 +1306,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root for _, targetPath := range ref.References() { targetPathParsed, err := dyn.NewPathFromString(targetPath) if err != nil { - return nil, fmt.Errorf("parsing %q: %w", targetPath, err) + return nil, nil, fmt.Errorf("parsing %q: %w", targetPath, err) } targetNodeDP, _ := config.GetNodeAndType(targetPathParsed) @@ -1288,7 +1332,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root lifecycleDeps, err := LifecycleDependsOn(configRoot, inputConfig) if err != nil { - return nil, fmt.Errorf("%s: %w", prefix, err) + return nil, nil, fmt.Errorf("%s: %w", prefix, err) } for _, dep := range lifecycleDeps { if !slices.Contains(dependsOn, dep) { @@ -1319,7 +1363,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root // are resolved before plan time, so SyncToJSON (called when cross-resource // refs are resolved during planNode) will never re-serialise these fields. if err := redactStruct(adapter, newStateConfig); err != nil { - return nil, fmt.Errorf("%s: cannot redact state: %w", node, err) + return nil, nil, fmt.Errorf("%s: cannot redact state: %w", node, err) } newState := &structvar.StructVar{ @@ -1330,7 +1374,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root // Convert to JSON for serialization in plan (values already redacted above). newStateJSON, err := newState.ToJSON() if err != nil { - return nil, fmt.Errorf("%s: cannot serialize state: %w", node, err) + return nil, nil, fmt.Errorf("%s: cannot serialize state: %w", node, err) } e := deployplan.PlanEntry{ @@ -1360,7 +1404,31 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root } } - return p, nil + return p, unmanagedPermissions, nil +} + +// unmanagedPermissionsNodes returns the ".permissions" nodes of deployed resources whose +// config and state have no permissions. A resource key that cannot be referenced (e.g. one with +// spaces) is left out: the node's object_id is a reference to the resource's ID. +func unmanagedPermissionsNodes(nodes []string, db *dstate.Database) map[string]bool { + result := make(map[string]bool) + for _, node := range nodes { + if !dresources.HasObjectPermissions(config.GetResourceTypeFromKey(node)) { + continue + } + if _, deployed := db.State[node]; !deployed { + continue + } + if !dynvar.IsPureVariableReference("${" + node + ".id}") { + continue + } + permissionsNode := node + ".permissions" + if _, ok := db.State[permissionsNode]; ok || slices.Contains(nodes, permissionsNode) { + continue + } + result[permissionsNode] = true + } + return result } // ExtractReferences extracts variable references from the config subtree rooted at node, diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 2d051dc21b9..9ac52b3282c 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/structs/structdiff" @@ -795,3 +796,30 @@ func TestLadderBackendDefaultBeforeRemoteAddition(t *testing.T) { assert.Equal(t, deployplan.Skip, changes["enable_elastic_disk"].Action) assert.Equal(t, deployplan.ReasonBackendDefault, changes["enable_elastic_disk"].Reason) } + +func TestUnmanagedPermissionsNodes(t *testing.T) { + db := &dstate.Database{State: map[string]dstate.ResourceEntry{ + "resources.jobs.unmanaged": {ID: "1"}, + "resources.jobs.in_state": {ID: "2"}, + "resources.jobs.in_state.permissions": {ID: "/jobs/2"}, + "resources.jobs.in_config": {ID: "3"}, + "resources.pipelines.unmanaged": {ID: "4"}, + "resources.secret_scopes.scope": {ID: "scope"}, + "resources.jobs.with spaces": {ID: "5"}, + }} + nodes := []string{ + "resources.jobs.unmanaged", + "resources.jobs.in_state", + "resources.jobs.in_config", + "resources.jobs.in_config.permissions", + "resources.jobs.not_deployed", + "resources.pipelines.unmanaged", + "resources.secret_scopes.scope", + "resources.jobs.with spaces", + } + + assert.Equal(t, map[string]bool{ + "resources.jobs.unmanaged.permissions": true, + "resources.pipelines.unmanaged.permissions": true, + }, unmanagedPermissionsNodes(nodes, db)) +} diff --git a/bundle/direct/dresources/permissions.go b/bundle/direct/dresources/permissions.go index 7f2eb0dbf04..f037ebf1ee8 100644 --- a/bundle/direct/dresources/permissions.go +++ b/bundle/direct/dresources/permissions.go @@ -32,6 +32,13 @@ var permissionResourceToObjectType = map[string]string{ "vector_search_endpoints": "/vector-search-endpoints/", } +// HasObjectPermissions reports whether permissions of the given resource type are managed +// through the permissions API. +func HasObjectPermissions(resourceType string) bool { + _, ok := permissionResourceToObjectType[resourceType] + return ok +} + type ResourcePermissions struct { client *databricks.WorkspaceClient diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 9c4d4b178cb..895185bcb16 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -356,7 +356,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } func RunPlan(ctx context.Context, b *bundle.Bundle) *deployplan.Plan { - plan, err := b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), &b.Config) + plan, err := b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), &b.Config, false) if err != nil { logdiag.LogError(ctx, err) return nil diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 07fed65ba80..8dcc53e6a39 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -323,7 +323,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle) { bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDestroy)) }() - plan, err := b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), nil) + plan, err := b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), nil, false) if err != nil { logdiag.LogError(ctx, err) return diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index 4fefd8504ac..276be13930b 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -200,7 +200,7 @@ func checkPlanOnTempState(ctx context.Context, b *bundle.Bundle, tempStatePath s return nil, fmt.Errorf("opening migrated state for plan check: %w", err) } - return planBundle.CalculatePlan(planCtx, b.WorkspaceClient(ctx), cfg) + return planBundle.CalculatePlan(planCtx, b.WorkspaceClient(ctx), cfg, false) } // recreatedResources returns the sorted keys of resources the plan would recreate diff --git a/cmd/bundle/config_remote_sync.go b/cmd/bundle/config_remote_sync.go index 83344855e5c..20e47808587 100644 --- a/cmd/bundle/config_remote_sync.go +++ b/cmd/bundle/config_remote_sync.go @@ -90,7 +90,7 @@ Examples: // ProcessBundleRet has already opened the direct state for reading. deployBundle := &b.DeploymentBundle - plan, err := deployBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), &b.Config) + plan, err := deployBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), &b.Config, true) if err != nil { stats.ErrorCategory = protos.BundleConfigRemoteSyncErrorCategoryDetectChangesFailed return fmt.Errorf("failed to detect changes: %w", err) From 93201ef5a4b03a46776979144d8fc4659fefcf0a Mon Sep 17 00:00:00 2001 From: "ilya.kuznetsov" Date: Fri, 13 Nov 2026 14:26:42 +0000 Subject: [PATCH 2/4] Read unmanaged permissions in configsync, sync declared permissions, keep other owners Co-authored-by: Isaac --- .../unmanaged_permissions/output.txt | 51 ++++- .../unmanaged_permissions/script | 29 ++- .../resourcemutator/fix_permissions.go | 15 +- .../resourcemutator/fix_permissions_test.go | 19 ++ bundle/configsync/diff.go | 87 +++----- bundle/configsync/diff_test.go | 97 +++++--- bundle/configsync/permissions.go | 209 ++++++++++++++++++ bundle/direct/bind.go | 4 +- bundle/direct/bundle_plan.go | 104 ++------- bundle/direct/bundle_plan_test.go | 28 --- bundle/direct/dresources/permissions.go | 7 - bundle/phases/deploy.go | 2 +- bundle/phases/destroy.go | 2 +- bundle/statemgmt/direct_migration.go | 2 +- cmd/bundle/config_remote_sync.go | 2 +- 15 files changed, 427 insertions(+), 231 deletions(-) create mode 100644 bundle/configsync/permissions.go diff --git a/acceptance/bundle/config-remote-sync/unmanaged_permissions/output.txt b/acceptance/bundle/config-remote-sync/unmanaged_permissions/output.txt index 3576a986551..a7b196b3cfc 100644 --- a/acceptance/bundle/config-remote-sync/unmanaged_permissions/output.txt +++ b/acceptance/bundle/config-remote-sync/unmanaged_permissions/output.txt @@ -32,10 +32,12 @@ Resource: resources.pipelines.my_pipeline + user_name: admin@example.com pipelines: my_pipeline: -@@ -20,4 +25,7 @@ +@@ -20,4 +25,9 @@ - notebook: path: /Users/{{workspace_user_name}}/notebook + permissions: ++ - level: IS_OWNER ++ user_name: owner@example.com + - group_name: users + level: CAN_VIEW @@ -45,6 +47,53 @@ Resource: resources.pipelines.my_pipeline No changes detected. +=== Deploy keeps the remote owner +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created jobs.my_job.permissions +Created pipelines.my_pipeline.permissions +Files: 1 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 2 unchanged + +>>> [CLI] permissions get pipelines [MY_PIPELINE_ID] +[{"user_name":"owner@example.com","group_name":null,"level":"IS_OWNER"},{"user_name":null,"group_name":"users","level":"CAN_VIEW"},{"user_name":"[USERNAME]","group_name":null,"level":"CAN_MANAGE"}] + +=== Change a level, add and remove principals remotely + +=== Detect and save changes +Detected changes in 1 resource(s): + +Resource: resources.jobs.my_job + permissions[group_name='data-team']: add + permissions[user_name='admin@example.com']: remove + permissions[user_name='viewer@example.com'].level: replace + + + +=== Configuration changes + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -15,8 +15,8 @@ + + permissions: +- - level: CAN_MANAGE ++ - level: CAN_VIEW + user_name: viewer@example.com +- - level: CAN_MANAGE_RUN +- user_name: admin@example.com ++ - group_name: data-team ++ level: CAN_MANAGE_RUN + pipelines: + my_pipeline: + +=== Deploy and sync again +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 4 unchanged +No changes detected. + + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.my_job diff --git a/acceptance/bundle/config-remote-sync/unmanaged_permissions/script b/acceptance/bundle/config-remote-sync/unmanaged_permissions/script index c69f5fe5d23..8f164b0ae0e 100644 --- a/acceptance/bundle/config-remote-sync/unmanaged_permissions/script +++ b/acceptance/bundle/config-remote-sync/unmanaged_permissions/script @@ -12,14 +12,16 @@ job_id="$(read_id.py my_job)" pipeline_id="$(read_id.py my_pipeline)" title "Add permissions remotely to resources that declare none" -# The owner entry is managed by deploy, so it must not be written to the config. +# The current user's owner entry is managed by deploy, so it must not be written to the config. $CLI permissions set jobs $job_id --json '{"access_control_list":[ {"permission_level":"IS_OWNER","user_name":"'"$CURRENT_USER_NAME"'"}, {"permission_level":"CAN_MANAGE","user_name":"viewer@example.com"}, {"permission_level":"CAN_MANAGE_RUN","user_name":"admin@example.com"} ]}' > /dev/null +# An owner transferred to another user is kept. $CLI permissions set pipelines $pipeline_id --json '{"access_control_list":[ - {"permission_level":"IS_OWNER","user_name":"'"$CURRENT_USER_NAME"'"}, + {"permission_level":"IS_OWNER","user_name":"owner@example.com"}, + {"permission_level":"CAN_MANAGE","user_name":"'"$CURRENT_USER_NAME"'"}, {"permission_level":"CAN_VIEW","group_name":"users"} ]}' > /dev/null @@ -33,3 +35,26 @@ rm databricks.yml.backup title "Sync again: the permissions are now in the config" $CLI bundle config-remote-sync + +title "Deploy keeps the remote owner" +$CLI bundle deploy +trace $CLI permissions get pipelines $pipeline_id | jq -c '[.access_control_list[] | {user_name, group_name, level: .all_permissions[0].permission_level}]' + +title "Change a level, add and remove principals remotely" +$CLI permissions set jobs $job_id --json '{"access_control_list":[ + {"permission_level":"IS_OWNER","user_name":"'"$CURRENT_USER_NAME"'"}, + {"permission_level":"CAN_VIEW","user_name":"viewer@example.com"}, + {"permission_level":"CAN_MANAGE_RUN","group_name":"data-team"} +]}' > /dev/null + +title "Detect and save changes" +cp databricks.yml databricks.yml.backup +$CLI bundle config-remote-sync --save + +title "Configuration changes" +trace diff.py databricks.yml.backup databricks.yml +rm databricks.yml.backup + +title "Deploy and sync again" +$CLI bundle deploy +$CLI bundle config-remote-sync diff --git a/bundle/config/mutator/resourcemutator/fix_permissions.go b/bundle/config/mutator/resourcemutator/fix_permissions.go index 9e6c785b0d9..7754c7f6e7b 100644 --- a/bundle/config/mutator/resourcemutator/fix_permissions.go +++ b/bundle/config/mutator/resourcemutator/fix_permissions.go @@ -2,6 +2,7 @@ package resourcemutator import ( "context" + "reflect" "strings" "github.com/databricks/cli/bundle" @@ -155,7 +156,7 @@ func useMaximumLevel(permissions dyn.Value, resourceType string) (dyn.Value, err } levelPerPrincipal := make(map[string]string) - principalIndex := make(map[string]int) + firstPermission := make(map[string]dyn.Value) var principals []string for _, permission := range permissionArray { @@ -168,10 +169,9 @@ func useMaximumLevel(permissions dyn.Value, resourceType string) (dyn.Value, err if principal == "" { continue } - _, ok = principalIndex[principal] + _, ok = firstPermission[principal] if !ok { - ind := len(principalIndex) - principalIndex[principal] = ind + firstPermission[principal] = permission principals = append(principals, principal) } levelPerPrincipal[principal] = resources.GetMaxLevel(levelPerPrincipal[principal], level) @@ -180,7 +180,12 @@ func useMaximumLevel(permissions dyn.Value, resourceType string) (dyn.Value, err var newPermissions []dyn.Value for _, principal := range principals { - newPermissions = append(newPermissions, createPermissionFromPrincipal(principal, levelPerPrincipal[principal])) + permission := createPermissionFromPrincipal(principal, levelPerPrincipal[principal]) + // Keep an unchanged entry as is: its source location lets config-remote-sync write it back. + if original := firstPermission[principal]; reflect.DeepEqual(original.AsAny(), permission.AsAny()) { + permission = original + } + newPermissions = append(newPermissions, permission) } return dyn.V(newPermissions), nil diff --git a/bundle/config/mutator/resourcemutator/fix_permissions_test.go b/bundle/config/mutator/resourcemutator/fix_permissions_test.go index f5488777ec4..fea459002ba 100644 --- a/bundle/config/mutator/resourcemutator/fix_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/fix_permissions_test.go @@ -4,7 +4,10 @@ import ( "testing" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestGetLevelScore(t *testing.T) { @@ -28,3 +31,19 @@ func TestGetMaxLevel(t *testing.T) { assert.Equal(t, "UNKNOWN_B", resources.GetMaxLevel("UNKNOWN_A", "UNKNOWN_B")) assert.Equal(t, "UNKNOWN_B", resources.GetMaxLevel("UNKNOWN_B", "UNKNOWN_A")) } + +func TestUseMaximumLevelKeepsUnchangedEntries(t *testing.T) { + loc := []diag.Location{{File: "databricks.yml", Line: 10}} + viewer := dyn.NewValue(map[string]dyn.Value{"level": dyn.V("CAN_VIEW"), "user_name": dyn.V("viewer@example.com")}, loc) + raised := dyn.NewValue(map[string]dyn.Value{"level": dyn.V("CAN_VIEW"), "group_name": dyn.V("users")}, loc) + duplicate := dyn.V(map[string]dyn.Value{"level": dyn.V("CAN_MANAGE"), "group_name": dyn.V("users")}) + + result, err := useMaximumLevel(dyn.V([]dyn.Value{viewer, raised, duplicate}), "jobs") + require.NoError(t, err) + + permissions := result.MustSequence() + require.Len(t, permissions, 2) + assert.Equal(t, loc, permissions[0].Locations()) + assert.Empty(t, permissions[1].Locations()) + assert.Equal(t, "CAN_MANAGE", permissions[1].Get("level").MustString()) +} diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index 6567bf8ae3d..c4673c3b4b3 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -8,12 +8,10 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structdiff" - "github.com/databricks/databricks-sdk-go/service/iam" ) type OperationType string @@ -147,11 +145,10 @@ func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan for resourceKey, entry := range plan.Plan { // permissions and grants are emitted as their own plan keys // ("resources...permissions" / ".grants"; see splitResourcePath - // in bundle/direct/bundle_plan.go). config-remote-sync cannot write them back - // to YAML: their fields (e.g. object_id) are server-populated with no source - // location, and bundle-level permissions have no per-resource YAML node at all, - // so resolving them would fail the whole sync. Skip them instead; only the - // permissions of resources that declare none are added below, as a whole. + // in bundle/direct/bundle_plan.go). Their fields (e.g. object_id) are + // server-populated with no source location, so they cannot be written back + // as is; permissions are added below under their parent resource instead, + // and grants are skipped. if isPermissionsOrGrantsSubResource(resourceKey) { continue } @@ -160,26 +157,13 @@ func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan if entry.Changes != nil { for path, changeDesc := range entry.Changes { - if changeDesc.Action == deployplan.Skip && changeDesc.Reason != deployplan.ReasonRemoteAddition { - continue - } - - fullPath := resourceKey + "." + path - change, err := convertChangeDesc(fullPath, changeDesc) + change, err := convertPlanChange(resourceKey+"."+path, changeDesc, b.Config.Presets.NamePrefix) if err != nil { return nil, fmt.Errorf("failed to compute config change for path %s: %w", path, err) } - if change.Operation == OperationSkip { - continue - } - // The state snapshot holds real per-field values, so New != Old - // means the local config diverged from the last deploy and this - // change overwrites that pending edit. - if !structdiff.IsEqual(changeDesc.Old, changeDesc.New) { - change.LocalEdit = true + if change != nil { + resourceChanges[path] = change } - change.Value = stripNamePrefix(fullPath, change.Value, b.Config.Presets.NamePrefix) - resourceChanges[path] = change } } @@ -190,52 +174,33 @@ func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan log.Debugf(ctx, "Resource %s has %d changes", resourceKey, len(resourceChanges)) } - for resourceKey, entry := range plan.Plan { - permissions := unmanagedPermissions(b, resourceKey, entry) - if len(permissions) == 0 { - continue - } - value, err := normalizeValue(permissions) - if err != nil { - return nil, fmt.Errorf("failed to compute config change for %s: %w", resourceKey, err) - } - parentKey := strings.TrimSuffix(resourceKey, ".permissions") - if changes[parentKey] == nil { - changes[parentKey] = make(ResourceChanges) - } - changes[parentKey]["permissions"] = &ConfigChangeDesc{ - Operation: OperationAdd, - Value: value, - } + if err := addPermissionsChanges(ctx, changes, b, plan); err != nil { + return nil, err } return changes, nil } -// unmanagedPermissions returns the remote permissions the plan read for a resource whose config -// declares none (see CalculatePlan), without the owner and the current user's CAN_MANAGE: -// FixPermissions adds those on deploy, so they do not belong in the config. -func unmanagedPermissions(b *bundle.Bundle, resourceKey string, entry *deployplan.PlanEntry) []dresources.StatePermission { - remote, ok := entry.RemoteState.(*dresources.PermissionsState) - if !ok { - return nil +// convertPlanChange converts a plan change at fullPath, returning nil when there is nothing to +// write back. +func convertPlanChange(fullPath string, changeDesc *deployplan.ChangeDesc, namePrefix string) (*ConfigChangeDesc, error) { + if changeDesc.Action == deployplan.Skip && changeDesc.Reason != deployplan.ReasonRemoteAddition { + return nil, nil } - path, err := dyn.NewPathFromString(resourceKey) + + change, err := convertChangeDesc(fullPath, changeDesc) if err != nil { - return nil + return nil, err } - if _, err := dyn.GetByPath(b.Config.Value(), path); err == nil { - return nil + if change.Operation == OperationSkip { + return nil, nil } - - currentUser := b.Config.Workspace.CurrentUser.UserName - var result []dresources.StatePermission - for _, p := range remote.EmbeddedSlice { - isCurrentUser := p.UserName == currentUser || p.ServicePrincipalName == currentUser - if p.Level == iam.PermissionLevelIsOwner || (p.Level == iam.PermissionLevelCanManage && isCurrentUser) { - continue - } - result = append(result, p) + // The state snapshot holds real per-field values, so New != Old + // means the local config diverged from the last deploy and this + // change overwrites that pending edit. + if !structdiff.IsEqual(changeDesc.Old, changeDesc.New) { + change.LocalEdit = true } - return result + change.Value = stripNamePrefix(fullPath, change.Value, namePrefix) + return change, nil } diff --git a/bundle/configsync/diff_test.go b/bundle/configsync/diff_test.go index 181aaa4e19c..13384a62997 100644 --- a/bundle/configsync/diff_test.go +++ b/bundle/configsync/diff_test.go @@ -1,14 +1,14 @@ package configsync import ( + "strings" "testing" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/jsonloader" + "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -304,52 +304,79 @@ func TestExtractChangesSkipsPermissionsAndGrants(t *testing.T) { assert.Contains(t, namedPerms, "description") } -func TestExtractChangesAddsUnmanagedPermissions(t *testing.T) { - value, err := jsonloader.LoadJSON([]byte(`{"resources": {"jobs": { - "unmanaged": {"name": "unmanaged"}, - "managed": {"name": "managed", "permissions": [{"level": "CAN_VIEW", "group_name": "users"}]} - }}}`), "test config") +func TestExtractChangesDeclaredPermissions(t *testing.T) { + value, err := yamlloader.LoadYAML("databricks.yml", strings.NewReader(` +resources: + jobs: + my_job: + permissions: + - level: CAN_VIEW + user_name: viewer@example.com + - level: CAN_VIEW + user_name: gone@example.com +`)) + require.NoError(t, err) + // FixPermissions adds the current user's entry, with no location. + value, err = dyn.Set(value, "resources.jobs.my_job.permissions", dyn.V(append( + value.Get("resources").Get("jobs").Get("my_job").Get("permissions").MustSequence(), + dyn.V(map[string]dyn.Value{"level": dyn.V("IS_OWNER"), "user_name": dyn.V("me@example.com")}), + ))) require.NoError(t, err) b := &bundle.Bundle{} require.NoError(t, b.Config.Mutate(func(dyn.Value) (dyn.Value, error) { return value, nil })) - b.Config.Workspace.CurrentUser = &config.User{User: &iam.User{UserName: "me@example.com"}} - remote := func(permissions ...dresources.StatePermission) *dresources.PermissionsState { - return &dresources.PermissionsState{ObjectID: "/jobs/123", EmbeddedSlice: permissions} - } plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ - "resources.jobs.unmanaged": {}, - "resources.jobs.unmanaged.permissions": { - Action: deployplan.Skip, - RemoteState: remote( - dresources.StatePermission{Level: iam.PermissionLevelIsOwner, UserName: "me@example.com"}, - dresources.StatePermission{Level: iam.PermissionLevelCanManage, UserName: "me@example.com"}, - dresources.StatePermission{Level: iam.PermissionLevelCanView, GroupName: "users"}, - dresources.StatePermission{Level: iam.PermissionLevelCanManage, UserName: "other@example.com"}, - ), - }, - "resources.jobs.owner_only.permissions": { - Action: deployplan.Skip, - RemoteState: remote(dresources.StatePermission{Level: iam.PermissionLevelIsOwner, UserName: "me@example.com"}), - }, - "resources.jobs.managed": {}, - "resources.jobs.managed.permissions": { - Action: deployplan.Skip, - RemoteState: remote(dresources.StatePermission{Level: iam.PermissionLevelCanManageRun, GroupName: "admins"}), + "resources.jobs.my_job": {}, + "resources.jobs.my_job.permissions": { + Action: deployplan.Update, + RemoteState: &dresources.PermissionsState{ObjectID: "/jobs/123"}, + Changes: deployplan.Changes{ + "[user_name='viewer@example.com'].level": { + Action: deployplan.Update, Old: "CAN_VIEW", New: "CAN_VIEW", Remote: "CAN_MANAGE", + }, + "[group_name='data-team']": { + Action: deployplan.Update, + Remote: dresources.StatePermission{Level: iam.PermissionLevelCanView, GroupName: "data-team"}, + }, + "[user_name='gone@example.com']": { + Action: deployplan.Update, + Old: dresources.StatePermission{Level: iam.PermissionLevelCanView, UserName: "gone@example.com"}, + New: dresources.StatePermission{Level: iam.PermissionLevelCanView, UserName: "gone@example.com"}, + }, + "[user_name='me@example.com'].level": { + Action: deployplan.Update, Old: "IS_OWNER", New: "IS_OWNER", Remote: "CAN_MANAGE", + }, + "object_id": {Action: deployplan.Update, Remote: "/jobs/456"}, + }, }, }} changes, err := ExtractChanges(t.Context(), b, plan) require.NoError(t, err) assert.Equal(t, Changes{ - "resources.jobs.unmanaged": { - "permissions": { + "resources.jobs.my_job": { + "permissions[user_name='viewer@example.com'].level": {Operation: OperationReplace, Value: "CAN_MANAGE"}, + "permissions[group_name='data-team']": { Operation: OperationAdd, - Value: []any{ - map[string]any{"level": "CAN_VIEW", "group_name": "users"}, - map[string]any{"level": "CAN_MANAGE", "user_name": "other@example.com"}, - }, + Value: map[string]any{"level": "CAN_VIEW", "group_name": "data-team"}, }, + "permissions[user_name='gone@example.com']": {Operation: OperationRemove}, }, }, changes) } + +func TestWithoutCurrentUserManage(t *testing.T) { + permissions := []dresources.StatePermission{ + {Level: iam.PermissionLevelIsOwner, UserName: "me@example.com"}, + {Level: iam.PermissionLevelCanManage, ServicePrincipalName: "me@example.com"}, + {Level: iam.PermissionLevelIsOwner, UserName: "owner@example.com"}, + {Level: iam.PermissionLevelCanManage, UserName: "other@example.com"}, + {Level: iam.PermissionLevelCanView, GroupName: "users"}, + } + + assert.Equal(t, []dresources.StatePermission{ + {Level: iam.PermissionLevelIsOwner, UserName: "owner@example.com"}, + {Level: iam.PermissionLevelCanManage, UserName: "other@example.com"}, + {Level: iam.PermissionLevelCanView, GroupName: "users"}, + }, withoutCurrentUserManage(permissions, "me@example.com")) +} diff --git a/bundle/configsync/permissions.go b/bundle/configsync/permissions.go new file mode 100644 index 00000000000..9098b317dd6 --- /dev/null +++ b/bundle/configsync/permissions.go @@ -0,0 +1,209 @@ +package configsync + +import ( + "context" + "fmt" + "reflect" + "slices" + "strings" + "sync" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/service/iam" + "golang.org/x/sync/errgroup" +) + +const permissionsReadLimit = 10 + +// addPermissionsChanges adds remote permission changes to the permissions field of their +// resource: per principal where the config declares permissions, and as a whole for deployed +// resources that declare none. +func addPermissionsChanges(ctx context.Context, changes Changes, b *bundle.Bundle, plan *deployplan.Plan) error { + for resourceKey, entry := range plan.Plan { + if !isPermissionsOrGrantsSubResource(resourceKey) { + continue + } + if err := addDeclaredPermissionsChanges(changes, b, resourceKey, entry); err != nil { + return err + } + } + + for resourceKey, permissions := range readUnmanagedPermissions(ctx, b, plan) { + value, err := normalizeValue(permissions) + if err != nil { + return fmt.Errorf("failed to compute config change for %s: %w", resourceKey, err) + } + changes.set(resourceKey, "permissions", &ConfigChangeDesc{ + Operation: OperationAdd, + Value: value, + }) + } + return nil +} + +// addDeclaredPermissionsChanges turns the plan's changes to a resource's permissions into +// changes of its permissions field, e.g. "[user_name='a'].level" becomes +// "permissions[user_name='a'].level". Only entries written in the resource's config are changed: +// the ones FixPermissions or bundle-level permissions add have no YAML node to write to. +func addDeclaredPermissionsChanges(changes Changes, b *bundle.Bundle, resourceKey string, entry *deployplan.PlanEntry) error { + if _, ok := entry.RemoteState.(*dresources.PermissionsState); !ok { + return nil + } + declared := declaredPrincipals(b, resourceKey) + if len(declared) == 0 { + return nil + } + parentKey := strings.TrimSuffix(resourceKey, ".permissions") + + for path, changeDesc := range entry.Changes { + node, err := structpath.ParsePath(path) + if err != nil { + return fmt.Errorf("failed to parse path %s: %w", path, err) + } + key, principal, ok := node.AsSlice()[0].KeyValue() + if !ok || (changeDesc.New != nil && !declared[key+"="+principal]) { + continue + } + + fieldPath := "permissions" + path + change, err := convertPlanChange(parentKey+"."+fieldPath, changeDesc, b.Config.Presets.NamePrefix) + if err != nil { + return fmt.Errorf("failed to compute config change for path %s: %w", resourceKey+"."+path, err) + } + if change != nil { + changes.set(parentKey, fieldPath, change) + } + } + return nil +} + +// declaredPrincipals returns the principals of the permissions written in the resource's config, +// as "=" like the plan's "[='']" selectors. Added entries have no location. +func declaredPrincipals(b *bundle.Bundle, resourceKey string) map[string]bool { + path, err := dyn.NewPathFromString(resourceKey) + if err != nil { + return nil + } + permissions, _ := dyn.GetByPath(b.Config.Value(), path) + sequence, _ := permissions.AsSequence() + result := make(map[string]bool) + for _, permission := range sequence { + if permission.Location().File == "" { + continue + } + for _, key := range []string{"user_name", "service_principal_name", "group_name"} { + if value, ok := permission.Get(key).AsString(); ok && value != "" { + result[key+"="+value] = true + } + } + } + return result +} + +func (c Changes) set(resourceKey, fieldPath string, change *ConfigChangeDesc) { + if c[resourceKey] == nil { + c[resourceKey] = make(ResourceChanges) + } + c[resourceKey][fieldPath] = change +} + +// readUnmanagedPermissions reads the remote permissions of deployed resources whose config and +// state have none, keyed by resource. They are imported as a whole: the current ACL cannot tell +// grants made before the deployment from the ones made since. +func readUnmanagedPermissions(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) map[string][]dresources.StatePermission { + var mu sync.Mutex + var g errgroup.Group + g.SetLimit(permissionsReadLimit) + result := make(map[string][]dresources.StatePermission) + currentUser := currentUserName(b) + + for resourceKey, entry := range plan.Plan { + permissionsKey := resourceKey + ".permissions" + if isPermissionsOrGrantsSubResource(resourceKey) || !entry.Action.KeepsID() || plan.Plan[permissionsKey] != nil { + continue + } + adapter := b.DeploymentBundle.Adapters[config.GetResourceTypeFromKey(permissionsKey)] + if adapter == nil || adapter.RemoteType() != reflect.TypeFor[*dresources.PermissionsState]() { + continue + } + + g.Go(func() error { + permissions, err := readPermissions(ctx, b, adapter, permissionsKey) + if err != nil { + // A user who can deploy a resource is not necessarily allowed to read its ACL. + log.Warnf(ctx, "reading permissions of %s: %s", resourceKey, err) + return nil + } + permissions = withoutCurrentUserManage(permissions, currentUser) + if len(permissions) > 0 { + mu.Lock() + result[resourceKey] = permissions + mu.Unlock() + } + return nil + }) + } + + _ = g.Wait() + return result +} + +func readPermissions(ctx context.Context, b *bundle.Bundle, adapter *dresources.Adapter, permissionsKey string) ([]dresources.StatePermission, error) { + sv, err := adapter.PrepareInputConfig([]dresources.StatePermission{}, permissionsKey) + if err != nil { + return nil, err + } + objectID := sv.Refs["object_id"] + ref, ok := dynvar.NewRef(dyn.V(objectID)) + if !ok { + // The resource key cannot be referenced, e.g. it has spaces. + return nil, nil + } + for _, target := range ref.References() { + path, err := structpath.ParsePath(target) + if err != nil { + return nil, err + } + value, err := b.DeploymentBundle.LookupReferencePreDeploy(ctx, path) + if err != nil { + return nil, fmt.Errorf("resolving %s: %w", target, err) + } + if err := sv.ResolveRef("${"+target+"}", value); err != nil { + return nil, err + } + } + + id := sv.Value.(*dresources.PermissionsState).ObjectID + remote, err := retry.With(ctx, retry.IsTransientRead, func() (any, error) { + return adapter.DoRead(ctx, id) + }) + if err != nil { + return nil, err + } + return remote.(*dresources.PermissionsState).EmbeddedSlice, nil +} + +// withoutCurrentUserManage drops the current user's IS_OWNER and CAN_MANAGE, one of which +// FixPermissions adds on deploy. Another principal's IS_OWNER is kept, so a transferred owner +// stays the owner after the next deploy. +func withoutCurrentUserManage(permissions []dresources.StatePermission, currentUser string) []dresources.StatePermission { + return slices.DeleteFunc(permissions, func(p dresources.StatePermission) bool { + isCurrentUser := p.UserName == currentUser || p.ServicePrincipalName == currentUser + return isCurrentUser && (p.Level == iam.PermissionLevelIsOwner || p.Level == iam.PermissionLevelCanManage) + }) +} + +func currentUserName(b *bundle.Bundle) string { + if b.Config.Workspace.CurrentUser == nil { + return "" + } + return b.Config.Workspace.CurrentUser.UserName +} diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index ce7306d856f..7fbf7ede662 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -124,7 +124,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac os.Remove(tmpStatePath) return nil, err } - plan, err := b.CalculatePlan(ctx, client, configRoot, false) + plan, err := b.CalculatePlan(ctx, client, configRoot) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -197,7 +197,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac os.Remove(tmpStatePath) return nil, err } - plan, err = b.CalculatePlan(ctx, client, configRoot, false) + plan, err = b.CalculatePlan(ctx, client, configRoot) if _, ferr := b.StateDB.Finalize(ctx); ferr != nil { log.Warnf(ctx, "failed to finalize state: %v", ferr) } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index cc5cb13afbb..237fcc75521 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -161,11 +161,7 @@ func (b *DeploymentBundle) StampDeploymentIdForFirstVersion(deploymentID string) // CalculatePlan computes the deployment plan by comparing local config against remote state. // StateDB must already be open for read before calling this function. -// -// With readUnmanagedPermissions, deployed resources whose config and state have no permissions -// also get a Skip ".permissions" entry holding the remote permissions in RemoteState. Only -// config-remote-sync sets it, to write remote permission changes back to the config. -func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, readUnmanagedPermissions bool) (*deployplan.Plan, error) { +func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root) (*deployplan.Plan, error) { b.StateDB.AssertOpenedForRead() err := b.init(client) @@ -173,7 +169,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return nil, err } - plan, unmanagedPermissions, err := b.makePlan(ctx, configRoot, &b.StateDB.Data, readUnmanagedPermissions) + plan, err := b.makePlan(ctx, configRoot, &b.StateDB.Data) if err != nil { return nil, fmt.Errorf("reading config: %w", err) } @@ -273,11 +269,6 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } - if unmanagedPermissions[resourceKey] { - b.readUnmanagedPermissions(ctx, adapter, resourceKey, entry) - return true - } - dbentry, hasEntry := b.StateDB.GetResourceEntry(resourceKey) // Tolerate empty-ID entries from older partial-recreate failures // (apply.Recreate now deletes state on the way through, but pre-fix @@ -424,29 +415,6 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return plan, nil } -// readUnmanagedPermissions records the remote permissions of a resource whose config declares -// none. There is nothing to deploy for it, so a failed read is not fatal: the user may not be -// allowed to read the permissions of a resource they can deploy. -func (b *DeploymentBundle) readUnmanagedPermissions(ctx context.Context, adapter *dresources.Adapter, resourceKey string, entry *deployplan.PlanEntry) { - entry.Action = deployplan.Skip - - sv, _ := b.StateCache.Load(resourceKey) - if len(sv.Refs) > 0 { - // The parent's ID is only known after deploy, e.g. it is recreated. - return - } - - id := sv.Value.(*dresources.PermissionsState).ObjectID - remoteState, err := retry.With(ctx, retry.IsTransientRead, func() (any, error) { - return adapter.DoRead(ctx, id) - }) - if err != nil { - log.Warnf(ctx, "reading permissions of %s id=%q: %s", resourceKey, id, err) - return - } - entry.RemoteState = remoteState -} - func getMaxAction(m map[string]*deployplan.ChangeDesc) deployplan.ActionType { result := deployplan.Skip for _, ch := range m { @@ -1181,7 +1149,7 @@ func (b *DeploymentBundle) resolveReferences(ctx context.Context, resourceKey st return true } -func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root, db *dstate.Database, readUnmanagedPermissions bool) (*deployplan.Plan, map[string]bool, error) { +func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root, db *dstate.Database) (*deployplan.Plan, error) { p := deployplan.NewPlanDirect() // Copy state metadata to plan for validation during apply @@ -1221,17 +1189,11 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root }, ) if err != nil { - return nil, nil, err + return nil, err } } } - var unmanagedPermissions map[string]bool - if readUnmanagedPermissions { - unmanagedPermissions = unmanagedPermissionsNodes(nodes, db) - nodes = slices.AppendSeq(nodes, maps.Keys(unmanagedPermissions)) - } - slices.Sort(nodes) for _, node := range nodes { delete(existingKeys, node) @@ -1239,25 +1201,22 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root prefix := "cannot plan " + node inputConfig, err := configRoot.GetResourceConfig(node) if err != nil { - return nil, nil, err - } - if unmanagedPermissions[node] { - inputConfig = reflect.New(config.ResourcesTypes()[config.GetResourceTypeFromKey(node)]).Interface() + return nil, err } adapter, err := b.getAdapterForKey(node) if err != nil { - return nil, nil, fmt.Errorf("%s: %w", prefix, err) + return nil, fmt.Errorf("%s: %w", prefix, err) } inputStructVar, err := adapter.PrepareInputConfig(inputConfig, node) if err != nil { - return nil, nil, fmt.Errorf("%s: %w", prefix, err) + return nil, fmt.Errorf("%s: %w", prefix, err) } newStateConfig, err := adapter.PrepareState(inputStructVar.Value) if err != nil { - return nil, nil, fmt.Errorf("%s: %w", prefix, err) + return nil, fmt.Errorf("%s: %w", prefix, err) } // Unescape "$${...}" to a literal "${...}" in the typed state, which is what @@ -1267,7 +1226,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root // extractReferences below still sees the escaped form and does not mistake // these placeholders for bundle references. if err := unescapeRefs(newStateConfig); err != nil { - return nil, nil, fmt.Errorf("%s: %w", prefix, err) + return nil, fmt.Errorf("%s: %w", prefix, err) } // New nodes only: a node with state must stay in the plan, otherwise emptying it plans nothing. @@ -1275,7 +1234,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root if _, hasState := db.State[node]; !hasState { empty, err := adapter.IsEmptyState(newStateConfig) if err != nil { - return nil, nil, fmt.Errorf("%s: %w", prefix, err) + return nil, fmt.Errorf("%s: %w", prefix, err) } if empty { continue @@ -1286,12 +1245,9 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root // This means input and state must be compatible: input can have more fields, but existing fields should not be moved // This means one cannot refer to fields not present in state (e.g. ${resources.jobs.foo.permissions}) - refs := make(map[string]string) - if !unmanagedPermissions[node] { - refs, err = extractReferences(configRoot.Value(), node, adapter.StateType()) - if err != nil { - return nil, nil, fmt.Errorf("failed to read references from config for %s: %w", node, err) - } + refs, err := extractReferences(configRoot.Value(), node, adapter.StateType()) + if err != nil { + return nil, fmt.Errorf("failed to read references from config for %s: %w", node, err) } maps.Copy(refs, inputStructVar.Refs) @@ -1306,7 +1262,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root for _, targetPath := range ref.References() { targetPathParsed, err := dyn.NewPathFromString(targetPath) if err != nil { - return nil, nil, fmt.Errorf("parsing %q: %w", targetPath, err) + return nil, fmt.Errorf("parsing %q: %w", targetPath, err) } targetNodeDP, _ := config.GetNodeAndType(targetPathParsed) @@ -1332,7 +1288,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root lifecycleDeps, err := LifecycleDependsOn(configRoot, inputConfig) if err != nil { - return nil, nil, fmt.Errorf("%s: %w", prefix, err) + return nil, fmt.Errorf("%s: %w", prefix, err) } for _, dep := range lifecycleDeps { if !slices.Contains(dependsOn, dep) { @@ -1363,7 +1319,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root // are resolved before plan time, so SyncToJSON (called when cross-resource // refs are resolved during planNode) will never re-serialise these fields. if err := redactStruct(adapter, newStateConfig); err != nil { - return nil, nil, fmt.Errorf("%s: cannot redact state: %w", node, err) + return nil, fmt.Errorf("%s: cannot redact state: %w", node, err) } newState := &structvar.StructVar{ @@ -1374,7 +1330,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root // Convert to JSON for serialization in plan (values already redacted above). newStateJSON, err := newState.ToJSON() if err != nil { - return nil, nil, fmt.Errorf("%s: cannot serialize state: %w", node, err) + return nil, fmt.Errorf("%s: cannot serialize state: %w", node, err) } e := deployplan.PlanEntry{ @@ -1404,31 +1360,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root } } - return p, unmanagedPermissions, nil -} - -// unmanagedPermissionsNodes returns the ".permissions" nodes of deployed resources whose -// config and state have no permissions. A resource key that cannot be referenced (e.g. one with -// spaces) is left out: the node's object_id is a reference to the resource's ID. -func unmanagedPermissionsNodes(nodes []string, db *dstate.Database) map[string]bool { - result := make(map[string]bool) - for _, node := range nodes { - if !dresources.HasObjectPermissions(config.GetResourceTypeFromKey(node)) { - continue - } - if _, deployed := db.State[node]; !deployed { - continue - } - if !dynvar.IsPureVariableReference("${" + node + ".id}") { - continue - } - permissionsNode := node + ".permissions" - if _, ok := db.State[permissionsNode]; ok || slices.Contains(nodes, permissionsNode) { - continue - } - result[permissionsNode] = true - } - return result + return p, nil } // ExtractReferences extracts variable references from the config subtree rooted at node, diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 9ac52b3282c..2d051dc21b9 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" - "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/structs/structdiff" @@ -796,30 +795,3 @@ func TestLadderBackendDefaultBeforeRemoteAddition(t *testing.T) { assert.Equal(t, deployplan.Skip, changes["enable_elastic_disk"].Action) assert.Equal(t, deployplan.ReasonBackendDefault, changes["enable_elastic_disk"].Reason) } - -func TestUnmanagedPermissionsNodes(t *testing.T) { - db := &dstate.Database{State: map[string]dstate.ResourceEntry{ - "resources.jobs.unmanaged": {ID: "1"}, - "resources.jobs.in_state": {ID: "2"}, - "resources.jobs.in_state.permissions": {ID: "/jobs/2"}, - "resources.jobs.in_config": {ID: "3"}, - "resources.pipelines.unmanaged": {ID: "4"}, - "resources.secret_scopes.scope": {ID: "scope"}, - "resources.jobs.with spaces": {ID: "5"}, - }} - nodes := []string{ - "resources.jobs.unmanaged", - "resources.jobs.in_state", - "resources.jobs.in_config", - "resources.jobs.in_config.permissions", - "resources.jobs.not_deployed", - "resources.pipelines.unmanaged", - "resources.secret_scopes.scope", - "resources.jobs.with spaces", - } - - assert.Equal(t, map[string]bool{ - "resources.jobs.unmanaged.permissions": true, - "resources.pipelines.unmanaged.permissions": true, - }, unmanagedPermissionsNodes(nodes, db)) -} diff --git a/bundle/direct/dresources/permissions.go b/bundle/direct/dresources/permissions.go index f037ebf1ee8..7f2eb0dbf04 100644 --- a/bundle/direct/dresources/permissions.go +++ b/bundle/direct/dresources/permissions.go @@ -32,13 +32,6 @@ var permissionResourceToObjectType = map[string]string{ "vector_search_endpoints": "/vector-search-endpoints/", } -// HasObjectPermissions reports whether permissions of the given resource type are managed -// through the permissions API. -func HasObjectPermissions(resourceType string) bool { - _, ok := permissionResourceToObjectType[resourceType] - return ok -} - type ResourcePermissions struct { client *databricks.WorkspaceClient diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 895185bcb16..9c4d4b178cb 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -356,7 +356,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } func RunPlan(ctx context.Context, b *bundle.Bundle) *deployplan.Plan { - plan, err := b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), &b.Config, false) + plan, err := b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), &b.Config) if err != nil { logdiag.LogError(ctx, err) return nil diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 8dcc53e6a39..07fed65ba80 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -323,7 +323,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle) { bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDestroy)) }() - plan, err := b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), nil, false) + plan, err := b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), nil) if err != nil { logdiag.LogError(ctx, err) return diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index 276be13930b..4fefd8504ac 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -200,7 +200,7 @@ func checkPlanOnTempState(ctx context.Context, b *bundle.Bundle, tempStatePath s return nil, fmt.Errorf("opening migrated state for plan check: %w", err) } - return planBundle.CalculatePlan(planCtx, b.WorkspaceClient(ctx), cfg, false) + return planBundle.CalculatePlan(planCtx, b.WorkspaceClient(ctx), cfg) } // recreatedResources returns the sorted keys of resources the plan would recreate diff --git a/cmd/bundle/config_remote_sync.go b/cmd/bundle/config_remote_sync.go index 20e47808587..83344855e5c 100644 --- a/cmd/bundle/config_remote_sync.go +++ b/cmd/bundle/config_remote_sync.go @@ -90,7 +90,7 @@ Examples: // ProcessBundleRet has already opened the direct state for reading. deployBundle := &b.DeploymentBundle - plan, err := deployBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), &b.Config, true) + plan, err := deployBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), &b.Config) if err != nil { stats.ErrorCategory = protos.BundleConfigRemoteSyncErrorCategoryDetectChangesFailed return fmt.Errorf("failed to detect changes: %w", err) From f958e79226d29de1a88c4e08e2bebab17d8bd040 Mon Sep 17 00:00:00 2001 From: "ilya.kuznetsov" Date: Fri, 13 Nov 2026 14:35:34 +0000 Subject: [PATCH 3/4] Resolve permission IDs of unchanged resources, import ACLs of removed permissions, keep the max-level entry location Co-authored-by: Isaac --- .../resourcemutator/fix_permissions.go | 15 +++-- .../resourcemutator/fix_permissions_test.go | 19 +++--- bundle/configsync/diff_test.go | 38 ++++++++++++ bundle/configsync/permissions.go | 58 ++++++++++++++----- 4 files changed, 100 insertions(+), 30 deletions(-) diff --git a/bundle/config/mutator/resourcemutator/fix_permissions.go b/bundle/config/mutator/resourcemutator/fix_permissions.go index 7754c7f6e7b..453cf905432 100644 --- a/bundle/config/mutator/resourcemutator/fix_permissions.go +++ b/bundle/config/mutator/resourcemutator/fix_permissions.go @@ -156,7 +156,7 @@ func useMaximumLevel(permissions dyn.Value, resourceType string) (dyn.Value, err } levelPerPrincipal := make(map[string]string) - firstPermission := make(map[string]dyn.Value) + maxPermission := make(map[string]dyn.Value) var principals []string for _, permission := range permissionArray { @@ -169,20 +169,23 @@ func useMaximumLevel(permissions dyn.Value, resourceType string) (dyn.Value, err if principal == "" { continue } - _, ok = firstPermission[principal] + _, ok = maxPermission[principal] if !ok { - firstPermission[principal] = permission principals = append(principals, principal) } - levelPerPrincipal[principal] = resources.GetMaxLevel(levelPerPrincipal[principal], level) + maxLevel := resources.GetMaxLevel(levelPerPrincipal[principal], level) + if maxLevel != levelPerPrincipal[principal] { + maxPermission[principal] = permission + } + levelPerPrincipal[principal] = maxLevel } var newPermissions []dyn.Value for _, principal := range principals { permission := createPermissionFromPrincipal(principal, levelPerPrincipal[principal]) - // Keep an unchanged entry as is: its source location lets config-remote-sync write it back. - if original := firstPermission[principal]; reflect.DeepEqual(original.AsAny(), permission.AsAny()) { + // Keep the entry that sets the level as is: its source location lets config-remote-sync write it back. + if original := maxPermission[principal]; reflect.DeepEqual(original.AsAny(), permission.AsAny()) { permission = original } newPermissions = append(newPermissions, permission) diff --git a/bundle/config/mutator/resourcemutator/fix_permissions_test.go b/bundle/config/mutator/resourcemutator/fix_permissions_test.go index fea459002ba..10cdb8a4d83 100644 --- a/bundle/config/mutator/resourcemutator/fix_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/fix_permissions_test.go @@ -32,18 +32,19 @@ func TestGetMaxLevel(t *testing.T) { assert.Equal(t, "UNKNOWN_B", resources.GetMaxLevel("UNKNOWN_B", "UNKNOWN_A")) } -func TestUseMaximumLevelKeepsUnchangedEntries(t *testing.T) { - loc := []diag.Location{{File: "databricks.yml", Line: 10}} - viewer := dyn.NewValue(map[string]dyn.Value{"level": dyn.V("CAN_VIEW"), "user_name": dyn.V("viewer@example.com")}, loc) - raised := dyn.NewValue(map[string]dyn.Value{"level": dyn.V("CAN_VIEW"), "group_name": dyn.V("users")}, loc) - duplicate := dyn.V(map[string]dyn.Value{"level": dyn.V("CAN_MANAGE"), "group_name": dyn.V("users")}) - - result, err := useMaximumLevel(dyn.V([]dyn.Value{viewer, raised, duplicate}), "jobs") +func TestUseMaximumLevelKeepsDeclaredEntries(t *testing.T) { + viewerLoc := []diag.Location{{File: "databricks.yml", Line: 10}} + managerLoc := []diag.Location{{File: "databricks.yml", Line: 14}} + viewer := dyn.NewValue(map[string]dyn.Value{"level": dyn.V("CAN_VIEW"), "user_name": dyn.V("viewer@example.com")}, viewerLoc) + lower := dyn.NewValue(map[string]dyn.Value{"level": dyn.V("CAN_VIEW"), "group_name": dyn.V("users")}, viewerLoc) + higher := dyn.NewValue(map[string]dyn.Value{"level": dyn.V("CAN_MANAGE"), "group_name": dyn.V("users")}, managerLoc) + + result, err := useMaximumLevel(dyn.V([]dyn.Value{viewer, lower, higher}), "jobs") require.NoError(t, err) permissions := result.MustSequence() require.Len(t, permissions, 2) - assert.Equal(t, loc, permissions[0].Locations()) - assert.Empty(t, permissions[1].Locations()) + assert.Equal(t, viewerLoc, permissions[0].Locations()) + assert.Equal(t, managerLoc, permissions[1].Locations()) assert.Equal(t, "CAN_MANAGE", permissions[1].Get("level").MustString()) } diff --git a/bundle/configsync/diff_test.go b/bundle/configsync/diff_test.go index 13384a62997..c7b081fca75 100644 --- a/bundle/configsync/diff_test.go +++ b/bundle/configsync/diff_test.go @@ -1,6 +1,8 @@ package configsync import ( + "maps" + "slices" "strings" "testing" @@ -9,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/yamlloader" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -380,3 +383,38 @@ func TestWithoutCurrentUserManage(t *testing.T) { {Level: iam.PermissionLevelCanView, GroupName: "users"}, }, withoutCurrentUserManage(permissions, "me@example.com")) } + +func TestUnmanagedPermissionsAdapters(t *testing.T) { + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) + b := &bundle.Bundle{} + b.DeploymentBundle.Adapters = adapters + + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.unmanaged": {Action: deployplan.Skip}, + "resources.jobs.declared": {Action: deployplan.Skip}, + "resources.jobs.declared.permissions": {Action: deployplan.Skip}, + "resources.jobs.removed": {Action: deployplan.Update}, + "resources.jobs.removed.permissions": {Action: deployplan.Delete}, + "resources.jobs.not_deployed": {Action: deployplan.Create}, + "resources.pipelines.unmanaged": {Action: deployplan.Skip}, + "resources.secret_scopes.scope": {Action: deployplan.Skip}, + "resources.schemas.schema": {Action: deployplan.Skip}, + "resources.schemas.schema.grants": {Action: deployplan.Skip}, + }} + + assert.ElementsMatch(t, + []string{"resources.jobs.unmanaged", "resources.jobs.removed", "resources.pipelines.unmanaged"}, + slices.Collect(maps.Keys(unmanagedPermissionsAdapters(b, plan))), + ) +} + +func TestLookupDeployedFieldReadsRemoteState(t *testing.T) { + b := &bundle.Bundle{} + b.DeploymentBundle.RemoteStateCache.Store("resources.model_serving_endpoints.my_endpoint", + &dresources.ModelServingEndpointRemote{EndpointId: "abc123"}) + + value, err := lookupDeployedField(b, structpath.MustParsePath("resources.model_serving_endpoints.my_endpoint.endpoint_id")) + require.NoError(t, err) + assert.Equal(t, "abc123", value) +} diff --git a/bundle/configsync/permissions.go b/bundle/configsync/permissions.go index 9098b317dd6..e28edbf42a5 100644 --- a/bundle/configsync/permissions.go +++ b/bundle/configsync/permissions.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/retry" + "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/iam" "golang.org/x/sync/errgroup" @@ -115,8 +116,8 @@ func (c Changes) set(resourceKey, fieldPath string, change *ConfigChangeDesc) { c[resourceKey][fieldPath] = change } -// readUnmanagedPermissions reads the remote permissions of deployed resources whose config and -// state have none, keyed by resource. They are imported as a whole: the current ACL cannot tell +// readUnmanagedPermissions reads the remote permissions of deployed resources whose config +// declares none, keyed by resource. They are imported as a whole: the current ACL cannot tell // grants made before the deployment from the ones made since. func readUnmanagedPermissions(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) map[string][]dresources.StatePermission { var mu sync.Mutex @@ -125,18 +126,9 @@ func readUnmanagedPermissions(ctx context.Context, b *bundle.Bundle, plan *deplo result := make(map[string][]dresources.StatePermission) currentUser := currentUserName(b) - for resourceKey, entry := range plan.Plan { - permissionsKey := resourceKey + ".permissions" - if isPermissionsOrGrantsSubResource(resourceKey) || !entry.Action.KeepsID() || plan.Plan[permissionsKey] != nil { - continue - } - adapter := b.DeploymentBundle.Adapters[config.GetResourceTypeFromKey(permissionsKey)] - if adapter == nil || adapter.RemoteType() != reflect.TypeFor[*dresources.PermissionsState]() { - continue - } - + for resourceKey, adapter := range unmanagedPermissionsAdapters(b, plan) { g.Go(func() error { - permissions, err := readPermissions(ctx, b, adapter, permissionsKey) + permissions, err := readPermissions(ctx, b, adapter, resourceKey+".permissions") if err != nil { // A user who can deploy a resource is not necessarily allowed to read its ACL. log.Warnf(ctx, "reading permissions of %s: %s", resourceKey, err) @@ -156,6 +148,27 @@ func readUnmanagedPermissions(ctx context.Context, b *bundle.Bundle, plan *deplo return result } +// unmanagedPermissionsAdapters returns the permissions adapter of every deployed resource whose +// config declares no permissions. A state-only permissions entry is deleted on the next deploy +// without changing the ACL, so it does not count. +func unmanagedPermissionsAdapters(b *bundle.Bundle, plan *deployplan.Plan) map[string]*dresources.Adapter { + result := make(map[string]*dresources.Adapter) + for resourceKey, entry := range plan.Plan { + if isPermissionsOrGrantsSubResource(resourceKey) || !entry.Action.KeepsID() { + continue + } + permissionsKey := resourceKey + ".permissions" + if permissions := plan.Plan[permissionsKey]; permissions != nil && permissions.Action != deployplan.Delete { + continue + } + adapter := b.DeploymentBundle.Adapters[config.GetResourceTypeFromKey(permissionsKey)] + if adapter != nil && adapter.RemoteType() == reflect.TypeFor[*dresources.PermissionsState]() { + result[resourceKey] = adapter + } + } + return result +} + func readPermissions(ctx context.Context, b *bundle.Bundle, adapter *dresources.Adapter, permissionsKey string) ([]dresources.StatePermission, error) { sv, err := adapter.PrepareInputConfig([]dresources.StatePermission{}, permissionsKey) if err != nil { @@ -172,7 +185,7 @@ func readPermissions(ctx context.Context, b *bundle.Bundle, adapter *dresources. if err != nil { return nil, err } - value, err := b.DeploymentBundle.LookupReferencePreDeploy(ctx, path) + value, err := lookupDeployedField(b, path) if err != nil { return nil, fmt.Errorf("resolving %s: %w", target, err) } @@ -191,12 +204,27 @@ func readPermissions(ctx context.Context, b *bundle.Bundle, adapter *dresources. return remote.(*dresources.PermissionsState).EmbeddedSlice, nil } +// lookupDeployedField reads a field of a deployed resource, such as the ID its permissions are +// keyed by. The plan drops the new state of unchanged resources, so other fields come from the +// remote state it read. +func lookupDeployedField(b *bundle.Bundle, path *structpath.PathNode) (any, error) { + resourceKey, field := path.Prefix(3).String(), path.SkipPrefix(3) + if field.String() == "id" { + return b.DeploymentBundle.StateDB.GetResourceID(resourceKey), nil + } + remote, ok := b.DeploymentBundle.RemoteStateCache.Load(resourceKey) + if !ok { + return nil, fmt.Errorf("no remote state for %s", resourceKey) + } + return structaccess.Get(remote, field) +} + // withoutCurrentUserManage drops the current user's IS_OWNER and CAN_MANAGE, one of which // FixPermissions adds on deploy. Another principal's IS_OWNER is kept, so a transferred owner // stays the owner after the next deploy. func withoutCurrentUserManage(permissions []dresources.StatePermission, currentUser string) []dresources.StatePermission { return slices.DeleteFunc(permissions, func(p dresources.StatePermission) bool { - isCurrentUser := p.UserName == currentUser || p.ServicePrincipalName == currentUser + isCurrentUser := currentUser != "" && (p.UserName == currentUser || p.ServicePrincipalName == currentUser) return isCurrentUser && (p.Level == iam.PermissionLevelIsOwner || p.Level == iam.PermissionLevelCanManage) }) } From b9f3db2e6f1e3b25c241f5efd30e2e98e5a4ec77 Mon Sep 17 00:00:00 2001 From: "ilya.kuznetsov" Date: Fri, 13 Nov 2026 14:42:06 +0000 Subject: [PATCH 4/4] Handle an empty remote ACL and resource keys with spaces Co-authored-by: Isaac --- bundle/configsync/diff_test.go | 37 +++++++++++++++++- bundle/configsync/permissions.go | 67 ++++++++++++++++---------------- 2 files changed, 69 insertions(+), 35 deletions(-) diff --git a/bundle/configsync/diff_test.go b/bundle/configsync/diff_test.go index c7b081fca75..8ea24dd5e04 100644 --- a/bundle/configsync/diff_test.go +++ b/bundle/configsync/diff_test.go @@ -11,7 +11,6 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -414,7 +413,41 @@ func TestLookupDeployedFieldReadsRemoteState(t *testing.T) { b.DeploymentBundle.RemoteStateCache.Store("resources.model_serving_endpoints.my_endpoint", &dresources.ModelServingEndpointRemote{EndpointId: "abc123"}) - value, err := lookupDeployedField(b, structpath.MustParsePath("resources.model_serving_endpoints.my_endpoint.endpoint_id")) + value, err := lookupDeployedField(b, "resources.model_serving_endpoints.my_endpoint", "endpoint_id") require.NoError(t, err) assert.Equal(t, "abc123", value) } + +func TestExtractChangesEmptyRemotePermissions(t *testing.T) { + value, err := yamlloader.LoadYAML("databricks.yml", strings.NewReader(` +resources: + apps: + my_app: + permissions: + - level: CAN_USE + group_name: users +`)) + require.NoError(t, err) + b := &bundle.Bundle{} + require.NoError(t, b.Config.Mutate(func(dyn.Value) (dyn.Value, error) { return value, nil })) + + declared := []dresources.StatePermission{{Level: "CAN_USE", GroupName: "users"}} + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.apps.my_app": {}, + "resources.apps.my_app.permissions": { + Action: deployplan.Update, + RemoteState: &dresources.PermissionsState{ObjectID: "/apps/my_app"}, + Changes: deployplan.Changes{ + "": {Action: deployplan.Update, Old: declared, New: declared}, + }, + }, + }} + + changes, err := ExtractChanges(t.Context(), b, plan) + require.NoError(t, err) + assert.Equal(t, Changes{ + "resources.apps.my_app": { + "permissions[group_name='users']": {Operation: OperationRemove}, + }, + }, changes) +} diff --git a/bundle/configsync/permissions.go b/bundle/configsync/permissions.go index e28edbf42a5..93c65f8aa9d 100644 --- a/bundle/configsync/permissions.go +++ b/bundle/configsync/permissions.go @@ -13,7 +13,6 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/structs/structaccess" @@ -29,9 +28,6 @@ const permissionsReadLimit = 10 // resources that declare none. func addPermissionsChanges(ctx context.Context, changes Changes, b *bundle.Bundle, plan *deployplan.Plan) error { for resourceKey, entry := range plan.Plan { - if !isPermissionsOrGrantsSubResource(resourceKey) { - continue - } if err := addDeclaredPermissionsChanges(changes, b, resourceKey, entry); err != nil { return err } @@ -69,8 +65,17 @@ func addDeclaredPermissionsChanges(changes Changes, b *bundle.Bundle, resourceKe if err != nil { return fmt.Errorf("failed to parse path %s: %w", path, err) } - key, principal, ok := node.AsSlice()[0].KeyValue() - if !ok || (changeDesc.New != nil && !declared[key+"="+principal]) { + nodes := node.AsSlice() + if len(nodes) == 0 { + // The remote ACL has no direct entries left, so the change covers the whole list. + if changeDesc.Action != deployplan.Skip { + for selector := range declared { + changes.set(parentKey, "permissions"+selector, &ConfigChangeDesc{Operation: OperationRemove}) + } + } + continue + } + if _, _, ok := nodes[0].KeyValue(); !ok || (changeDesc.New != nil && !declared[nodes[0].String()]) { continue } @@ -86,8 +91,8 @@ func addDeclaredPermissionsChanges(changes Changes, b *bundle.Bundle, resourceKe return nil } -// declaredPrincipals returns the principals of the permissions written in the resource's config, -// as "=" like the plan's "[='']" selectors. Added entries have no location. +// declaredPrincipals returns the permissions written in the resource's config as the plan's +// selectors, e.g. "[user_name='a']". Entries added by mutators have no location. func declaredPrincipals(b *bundle.Bundle, resourceKey string) map[string]bool { path, err := dyn.NewPathFromString(resourceKey) if err != nil { @@ -102,7 +107,7 @@ func declaredPrincipals(b *bundle.Bundle, resourceKey string) map[string]bool { } for _, key := range []string{"user_name", "service_principal_name", "group_name"} { if value, ok := permission.Get(key).AsString(); ok && value != "" { - result[key+"="+value] = true + result[structpath.NewKeyValue(nil, key, value).String()] = true } } } @@ -128,7 +133,7 @@ func readUnmanagedPermissions(ctx context.Context, b *bundle.Bundle, plan *deplo for resourceKey, adapter := range unmanagedPermissionsAdapters(b, plan) { g.Go(func() error { - permissions, err := readPermissions(ctx, b, adapter, resourceKey+".permissions") + permissions, err := readPermissions(ctx, b, adapter, resourceKey) if err != nil { // A user who can deploy a resource is not necessarily allowed to read its ACL. log.Warnf(ctx, "reading permissions of %s: %s", resourceKey, err) @@ -169,32 +174,25 @@ func unmanagedPermissionsAdapters(b *bundle.Bundle, plan *deployplan.Plan) map[s return result } -func readPermissions(ctx context.Context, b *bundle.Bundle, adapter *dresources.Adapter, permissionsKey string) ([]dresources.StatePermission, error) { - sv, err := adapter.PrepareInputConfig([]dresources.StatePermission{}, permissionsKey) +func readPermissions(ctx context.Context, b *bundle.Bundle, adapter *dresources.Adapter, resourceKey string) ([]dresources.StatePermission, error) { + sv, err := adapter.PrepareInputConfig([]dresources.StatePermission{}, resourceKey+".permissions") if err != nil { return nil, err } + // The object ID references a field of the resource, e.g. "/jobs/${resources.jobs.foo.id}". It is + // resolved here by key, since a key with e.g. spaces does not parse as a reference. objectID := sv.Refs["object_id"] - ref, ok := dynvar.NewRef(dyn.V(objectID)) - if !ok { - // The resource key cannot be referenced, e.g. it has spaces. - return nil, nil + objectType, field, ok := strings.Cut(objectID, "${"+resourceKey+".") + field, hasSuffix := strings.CutSuffix(field, "}") + if !ok || !hasSuffix { + return nil, fmt.Errorf("unexpected object_id %q", objectID) } - for _, target := range ref.References() { - path, err := structpath.ParsePath(target) - if err != nil { - return nil, err - } - value, err := lookupDeployedField(b, path) - if err != nil { - return nil, fmt.Errorf("resolving %s: %w", target, err) - } - if err := sv.ResolveRef("${"+target+"}", value); err != nil { - return nil, err - } + value, err := lookupDeployedField(b, resourceKey, field) + if err != nil { + return nil, fmt.Errorf("resolving %s.%s: %w", resourceKey, field, err) } - id := sv.Value.(*dresources.PermissionsState).ObjectID + id := objectType + fmt.Sprint(value) remote, err := retry.With(ctx, retry.IsTransientRead, func() (any, error) { return adapter.DoRead(ctx, id) }) @@ -207,16 +205,19 @@ func readPermissions(ctx context.Context, b *bundle.Bundle, adapter *dresources. // lookupDeployedField reads a field of a deployed resource, such as the ID its permissions are // keyed by. The plan drops the new state of unchanged resources, so other fields come from the // remote state it read. -func lookupDeployedField(b *bundle.Bundle, path *structpath.PathNode) (any, error) { - resourceKey, field := path.Prefix(3).String(), path.SkipPrefix(3) - if field.String() == "id" { +func lookupDeployedField(b *bundle.Bundle, resourceKey, field string) (any, error) { + if field == "id" { return b.DeploymentBundle.StateDB.GetResourceID(resourceKey), nil } + path, err := structpath.ParsePath(field) + if err != nil { + return nil, err + } remote, ok := b.DeploymentBundle.RemoteStateCache.Load(resourceKey) if !ok { return nil, fmt.Errorf("no remote state for %s", resourceKey) } - return structaccess.Get(remote, field) + return structaccess.Get(remote, path) } // withoutCurrentUserManage drops the current user's IS_OWNER and CAN_MANAGE, one of which