Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 0 additions & 15 deletions acceptance/bundle/config-remote-sync/job_fields/script
Original file line number Diff line number Diff line change
Expand Up @@ -40,21 +40,6 @@ if "tags" not in r:
r["tags"]["team"] = "data"
EOF

# TODO add support for permissions
# title "Modify job permissions remotely"
# current_user=$($CLI current-user me --output json | jq -r .userName)
# cat > permissions.json <<EOF
# {
# "access_control_list": [
# {"permission_level": "IS_OWNER", "user_name": "$current_user"},
# {"permission_level": "CAN_MANAGE", "user_name": "viewer@example.com"},
# {"permission_level": "CAN_MANAGE_RUN", "user_name": "admin@example.com"}
# ]
# }
# EOF
# $CLI permissions set jobs $job_id --json @permissions.json
# rm permissions.json

title "Detect and save changes"
cp databricks.yml databricks.yml.backup
$CLI bundle config-remote-sync --save
Expand Down
15 changes: 0 additions & 15 deletions acceptance/bundle/config-remote-sync/pipeline_fields/script
Original file line number Diff line number Diff line change
Expand Up @@ -26,21 +26,6 @@ r["tags"]["foo"] = "bar"
r.setdefault("environment", {}).setdefault("dependencies", []).append("--editable /Workspace/bar")
EOF

# TODO add support for permissions
# title "Modify pipeline permissions remotely"
# current_user=$($CLI current-user me --output json | jq -r .userName)
# cat > permissions.json <<EOF
# {
# "access_control_list": [
# {"permission_level": "IS_OWNER", "user_name": "$current_user"},
# {"permission_level": "CAN_MANAGE", "user_name": "viewer@example.com"},
# {"permission_level": "CAN_RUN", "user_name": "admin@example.com"}
# ]
# }
# EOF
# $CLI permissions set pipelines $pipeline_id --json @permissions.json
# rm permissions.json

title "Detect and save changes"
cp databricks.yml databricks.yml.backup
$CLI bundle config-remote-sync --save
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
bundle:
name: test-bundle-$UNIQUE_NAME

resources:
jobs:
my_job:
tasks:
- task_key: main
notebook_task:
notebook_path: /Users/{{workspace_user_name}}/main
new_cluster:
spark_version: $DEFAULT_SPARK_VERSION
node_type_id: $NODE_TYPE_ID
num_workers: 1

pipelines:
my_pipeline:
name: test-pipeline-$UNIQUE_NAME
libraries:
- notebook:
path: /Users/{{workspace_user_name}}/notebook

targets:
default:
mode: development
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Cloud = false
GOOS.windows = false
EnvMatrix.DMS = ["", "true"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
Created jobs.my_job
Created pipelines.my_pipeline
Files: 2 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

=== Add permissions remotely to resources that declare none

=== Detect and save changes
Detected changes in 2 resource(s):

Resource: resources.jobs.my_job
permissions: add

Resource: resources.pipelines.my_pipeline
permissions: add



=== Configuration changes

>>> diff.py databricks.yml.backup databricks.yml
--- databricks.yml.backup
+++ databricks.yml
@@ -14,4 +14,9 @@
num_workers: 1

+ permissions:
+ - level: CAN_MANAGE
+ user_name: viewer@example.com
+ - level: CAN_MANAGE_RUN
+ user_name: admin@example.com
pipelines:
my_pipeline:
@@ -20,4 +25,9 @@
- notebook:
path: /Users/{{workspace_user_name}}/notebook
+ permissions:
+ - level: IS_OWNER
+ user_name: owner@example.com
+ - group_name: users
+ level: CAN_VIEW

targets:

=== Sync again: the permissions are now in the config
No changes detected.


=== Deploy keeps the remote owner
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
Created jobs.my_job.permissions
Created pipelines.my_pipeline.permissions
Files: 1 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 2 unchanged

>>> [CLI] permissions get pipelines [MY_PIPELINE_ID]
[{"user_name":"owner@example.com","group_name":null,"level":"IS_OWNER"},{"user_name":null,"group_name":"users","level":"CAN_VIEW"},{"user_name":"[USERNAME]","group_name":null,"level":"CAN_MANAGE"}]

=== Change a level, add and remove principals remotely

=== Detect and save changes
Detected changes in 1 resource(s):

Resource: resources.jobs.my_job
permissions[group_name='data-team']: add
permissions[user_name='admin@example.com']: remove
permissions[user_name='viewer@example.com'].level: replace



=== Configuration changes

>>> diff.py databricks.yml.backup databricks.yml
--- databricks.yml.backup
+++ databricks.yml
@@ -15,8 +15,8 @@

permissions:
- - level: CAN_MANAGE
+ - level: CAN_VIEW
user_name: viewer@example.com
- - level: CAN_MANAGE_RUN
- user_name: admin@example.com
+ - group_name: data-team
+ level: CAN_MANAGE_RUN
pipelines:
my_pipeline:

=== Deploy and sync again
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
Files: 1 uploaded, 0 deleted
Resources: 0 created, 0 changed, 0 deleted, 4 unchanged
No changes detected.


>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
delete resources.jobs.my_job
delete resources.pipelines.my_pipeline

This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the
Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion:
delete resources.pipelines.my_pipeline

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default

Destroy: 2 deleted
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#!/bin/bash

envsubst < databricks.yml.tmpl > databricks.yml

cleanup() {
trace $CLI bundle destroy --auto-approve
}
trap cleanup EXIT

$CLI bundle deploy
job_id="$(read_id.py my_job)"
pipeline_id="$(read_id.py my_pipeline)"

title "Add permissions remotely to resources that declare none"
# The current user's owner entry is managed by deploy, so it must not be written to the config.
$CLI permissions set jobs $job_id --json '{"access_control_list":[
{"permission_level":"IS_OWNER","user_name":"'"$CURRENT_USER_NAME"'"},
{"permission_level":"CAN_MANAGE","user_name":"viewer@example.com"},
{"permission_level":"CAN_MANAGE_RUN","user_name":"admin@example.com"}
]}' > /dev/null
# An owner transferred to another user is kept.
$CLI permissions set pipelines $pipeline_id --json '{"access_control_list":[
{"permission_level":"IS_OWNER","user_name":"owner@example.com"},
{"permission_level":"CAN_MANAGE","user_name":"'"$CURRENT_USER_NAME"'"},
{"permission_level":"CAN_VIEW","group_name":"users"}
]}' > /dev/null

title "Detect and save changes"
cp databricks.yml databricks.yml.backup
$CLI bundle config-remote-sync --save

title "Configuration changes"
trace diff.py databricks.yml.backup databricks.yml
rm databricks.yml.backup

title "Sync again: the permissions are now in the config"
$CLI bundle config-remote-sync

title "Deploy keeps the remote owner"
$CLI bundle deploy
trace $CLI permissions get pipelines $pipeline_id | jq -c '[.access_control_list[] | {user_name, group_name, level: .all_permissions[0].permission_level}]'

title "Change a level, add and remove principals remotely"
$CLI permissions set jobs $job_id --json '{"access_control_list":[
{"permission_level":"IS_OWNER","user_name":"'"$CURRENT_USER_NAME"'"},
{"permission_level":"CAN_VIEW","user_name":"viewer@example.com"},
{"permission_level":"CAN_MANAGE_RUN","group_name":"data-team"}
]}' > /dev/null

title "Detect and save changes"
cp databricks.yml databricks.yml.backup
$CLI bundle config-remote-sync --save

title "Configuration changes"
trace diff.py databricks.yml.backup databricks.yml
rm databricks.yml.backup

title "Deploy and sync again"
$CLI bundle deploy
$CLI bundle config-remote-sync
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
RecordRequests = false
Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"]

Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true"
20 changes: 14 additions & 6 deletions bundle/config/mutator/resourcemutator/fix_permissions.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package resourcemutator

import (
"context"
"reflect"
"strings"

"github.com/databricks/cli/bundle"
Expand Down Expand Up @@ -155,7 +156,7 @@ func useMaximumLevel(permissions dyn.Value, resourceType string) (dyn.Value, err
}

levelPerPrincipal := make(map[string]string)
principalIndex := make(map[string]int)
maxPermission := make(map[string]dyn.Value)
var principals []string

for _, permission := range permissionArray {
Expand All @@ -168,19 +169,26 @@ func useMaximumLevel(permissions dyn.Value, resourceType string) (dyn.Value, err
if principal == "" {
continue
}
_, ok = principalIndex[principal]
_, ok = maxPermission[principal]
if !ok {
ind := len(principalIndex)
principalIndex[principal] = ind
principals = append(principals, principal)
}
levelPerPrincipal[principal] = resources.GetMaxLevel(levelPerPrincipal[principal], level)
maxLevel := resources.GetMaxLevel(levelPerPrincipal[principal], level)
if maxLevel != levelPerPrincipal[principal] {
maxPermission[principal] = permission
}
levelPerPrincipal[principal] = maxLevel
}

var newPermissions []dyn.Value

for _, principal := range principals {
newPermissions = append(newPermissions, createPermissionFromPrincipal(principal, levelPerPrincipal[principal]))
permission := createPermissionFromPrincipal(principal, levelPerPrincipal[principal])
// Keep the entry that sets the level as is: its source location lets config-remote-sync write it back.
if original := maxPermission[principal]; reflect.DeepEqual(original.AsAny(), permission.AsAny()) {
permission = original
}
newPermissions = append(newPermissions, permission)
}

return dyn.V(newPermissions), nil
Expand Down
20 changes: 20 additions & 0 deletions bundle/config/mutator/resourcemutator/fix_permissions_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@ import (
"testing"

"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestGetLevelScore(t *testing.T) {
Expand All @@ -28,3 +31,20 @@ func TestGetMaxLevel(t *testing.T) {
assert.Equal(t, "UNKNOWN_B", resources.GetMaxLevel("UNKNOWN_A", "UNKNOWN_B"))
assert.Equal(t, "UNKNOWN_B", resources.GetMaxLevel("UNKNOWN_B", "UNKNOWN_A"))
}

func TestUseMaximumLevelKeepsDeclaredEntries(t *testing.T) {
viewerLoc := []diag.Location{{File: "databricks.yml", Line: 10}}
managerLoc := []diag.Location{{File: "databricks.yml", Line: 14}}
viewer := dyn.NewValue(map[string]dyn.Value{"level": dyn.V("CAN_VIEW"), "user_name": dyn.V("viewer@example.com")}, viewerLoc)
lower := dyn.NewValue(map[string]dyn.Value{"level": dyn.V("CAN_VIEW"), "group_name": dyn.V("users")}, viewerLoc)
higher := dyn.NewValue(map[string]dyn.Value{"level": dyn.V("CAN_MANAGE"), "group_name": dyn.V("users")}, managerLoc)

result, err := useMaximumLevel(dyn.V([]dyn.Value{viewer, lower, higher}), "jobs")
require.NoError(t, err)

permissions := result.MustSequence()
require.Len(t, permissions, 2)
assert.Equal(t, viewerLoc, permissions[0].Locations())
assert.Equal(t, managerLoc, permissions[1].Locations())
assert.Equal(t, "CAN_MANAGE", permissions[1].Get("level").MustString())
}
Loading
Loading