Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
bundle:
name: test-bundle-$UNIQUE_NAME

resources:
jobs:
user_to_user:
name: user-to-user-$UNIQUE_NAME
run_as:
user_name: first-user@example.com
tasks:
- task_key: main
notebook_task:
notebook_path: /Users/{{workspace_user_name}}/notebook

user_to_creator:
name: user-to-creator-$UNIQUE_NAME
run_as:
user_name: first-user@example.com
tasks:
- task_key: main
notebook_task:
notebook_path: /Users/{{workspace_user_name}}/notebook

user_to_sp:
name: user-to-sp-$UNIQUE_NAME
run_as:
user_name: first-user@example.com
tasks:
- task_key: main
notebook_task:
notebook_path: /Users/{{workspace_user_name}}/notebook
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Cloud = false
GOOS.windows = false
EnvMatrix.DMS = ["", "true"]
73 changes: 73 additions & 0 deletions acceptance/bundle/config-remote-sync/job_run_as/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
Created jobs.user_to_creator
Created jobs.user_to_sp
Created jobs.user_to_user
Files: 2 uploaded, 0 deleted
Resources: 3 created, 0 changed, 0 deleted, 0 unchanged

=== Change run_as remotely

=== Detect and save changes

>>> [CLI] bundle config-remote-sync --save
Detected changes in 3 resource(s):

Resource: resources.jobs.user_to_creator
run_as.user_name: replace

Resource: resources.jobs.user_to_sp
run_as.service_principal_name: add
run_as.user_name: remove

Resource: resources.jobs.user_to_user
run_as.user_name: replace



>>> diff.py databricks.yml.backup databricks.yml
--- databricks.yml.backup
+++ databricks.yml
@@ -7,5 +7,5 @@
name: user-to-user-[UNIQUE_NAME]
run_as:
- user_name: first-user@example.com
+ user_name: second-user@example.com
tasks:
- task_key: main
@@ -16,5 +16,5 @@
name: user-to-creator-[UNIQUE_NAME]
run_as:
- user_name: first-user@example.com
+ user_name: [USERNAME]
tasks:
- task_key: main
@@ -25,5 +25,5 @@
name: user-to-sp-[UNIQUE_NAME]
run_as:
- user_name: first-user@example.com
+ service_principal_name: [UUID]
tasks:
- task_key: main

=== Second sync is a no-op

>>> [CLI] bundle config-remote-sync
No changes detected.


=== Plan after sync

>>> [CLI] bundle plan
Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged

>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
delete resources.jobs.user_to_creator
delete resources.jobs.user_to_sp
delete resources.jobs.user_to_user

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default

Destroy: 3 deleted
31 changes: 31 additions & 0 deletions acceptance/bundle/config-remote-sync/job_run_as/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
envsubst < databricks.yml.tmpl > databricks.yml

cleanup() {
trace $CLI bundle destroy --auto-approve
}
trap cleanup EXIT

trace $CLI bundle deploy

title "Change run_as remotely"
edit_resource.py jobs "$(read_id.py user_to_user)" <<'PY'
r["run_as"] = {"user_name": "second-user@example.com"}
PY
edit_resource.py jobs "$(read_id.py user_to_creator)" <<PY
r["run_as"] = {"user_name": "$CURRENT_USER_NAME"}
PY
edit_resource.py jobs "$(read_id.py user_to_sp)" <<'PY'
r["run_as"] = {"service_principal_name": "11111111-2222-3333-4444-555555555555"}
PY

title "Detect and save changes"
cp databricks.yml databricks.yml.backup
trace $CLI bundle config-remote-sync --save
trace diff.py databricks.yml.backup databricks.yml
rm databricks.yml.backup

title "Second sync is a no-op"
trace $CLI bundle config-remote-sync

title "Plan after sync"
trace $CLI bundle plan
3 changes: 3 additions & 0 deletions acceptance/bundle/config-remote-sync/job_run_as/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"]
# The principals in this test do not exist in a real workspace.
Cloud = false
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
bundle:
name: test-bundle-$UNIQUE_NAME

variables:
runner:
default: var-user@example.com

# Every job inherits this run_as unless it sets its own.
run_as:
user_name: bundle-user@example.com

resources:
jobs:
inherited_unchanged:
name: inherited-unchanged-$UNIQUE_NAME
tasks:
- task_key: main
notebook_task:
notebook_path: /Users/{{workspace_user_name}}/notebook

inherited_to_user:
name: inherited-to-user-$UNIQUE_NAME
tasks:
- task_key: main
notebook_task:
notebook_path: /Users/{{workspace_user_name}}/notebook

inherited_to_sp:
name: inherited-to-sp-$UNIQUE_NAME
tasks:
- task_key: main
notebook_task:
notebook_path: /Users/{{workspace_user_name}}/notebook

variable_to_user:
name: variable-to-user-$UNIQUE_NAME
run_as:
user_name: ${var.runner}
tasks:
- task_key: main
notebook_task:
notebook_path: /Users/{{workspace_user_name}}/notebook

target_override_to_group:
name: target-override-to-group-$UNIQUE_NAME
tasks:
- task_key: main
notebook_task:
notebook_path: /Users/{{workspace_user_name}}/notebook

targets:
default:
resources:
jobs:
target_override_to_group:
run_as:
user_name: target-user@example.com
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Cloud = false
GOOS.windows = false
EnvMatrix.DMS = ["", "true"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
Created jobs.inherited_to_sp
Created jobs.inherited_to_user
Created jobs.inherited_unchanged
Created jobs.target_override_to_group
Created jobs.variable_to_user
Files: 2 uploaded, 0 deleted
Resources: 5 created, 0 changed, 0 deleted, 0 unchanged

=== Change run_as remotely

=== Detect and save changes

>>> [CLI] bundle config-remote-sync --save
Detected changes in 4 resource(s):

Resource: resources.jobs.inherited_to_sp
run_as.service_principal_name: add
run_as.user_name: remove

Resource: resources.jobs.inherited_to_user
run_as.user_name: add

Resource: resources.jobs.target_override_to_group
run_as.group_name: add
run_as.user_name: remove

Resource: resources.jobs.variable_to_user
run_as.user_name: replace



>>> diff.py databricks.yml.backup databricks.yml
--- databricks.yml.backup
+++ databricks.yml
@@ -26,4 +26,6 @@
notebook_path: /Users/{{workspace_user_name}}/notebook

+ run_as:
+ user_name: second-user@example.com
inherited_to_sp:
name: inherited-to-sp-[UNIQUE_NAME]
@@ -33,8 +35,10 @@
notebook_path: /Users/{{workspace_user_name}}/notebook

+ run_as:
+ service_principal_name: [UUID]
variable_to_user:
name: variable-to-user-[UNIQUE_NAME]
run_as:
- user_name: ${var.runner}
+ user_name: second-user@example.com
tasks:
- task_key: main
@@ -55,3 +59,3 @@
target_override_to_group:
run_as:
- user_name: target-user@example.com
+ group_name: data-engineers

=== Second sync is a no-op

>>> [CLI] bundle config-remote-sync
No changes detected.


=== Plan after sync

>>> [CLI] bundle plan
Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged

>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
delete resources.jobs.inherited_to_sp
delete resources.jobs.inherited_to_user
delete resources.jobs.inherited_unchanged
delete resources.jobs.target_override_to_group
delete resources.jobs.variable_to_user

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default

Destroy: 5 deleted
34 changes: 34 additions & 0 deletions acceptance/bundle/config-remote-sync/job_run_as_sources/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
envsubst < databricks.yml.tmpl > databricks.yml

cleanup() {
trace $CLI bundle destroy --auto-approve
}
trap cleanup EXIT

trace $CLI bundle deploy

title "Change run_as remotely"
edit_resource.py jobs "$(read_id.py inherited_to_user)" <<'PY'
r["run_as"] = {"user_name": "second-user@example.com"}
PY
edit_resource.py jobs "$(read_id.py inherited_to_sp)" <<'PY'
r["run_as"] = {"service_principal_name": "11111111-2222-3333-4444-555555555555"}
PY
edit_resource.py jobs "$(read_id.py variable_to_user)" <<'PY'
r["run_as"] = {"user_name": "second-user@example.com"}
PY
edit_resource.py jobs "$(read_id.py target_override_to_group)" <<'PY'
r["run_as"] = {"group_name": "data-engineers"}
PY

title "Detect and save changes"
cp databricks.yml databricks.yml.backup
trace $CLI bundle config-remote-sync --save
trace diff.py databricks.yml.backup databricks.yml
rm databricks.yml.backup

title "Second sync is a no-op"
trace $CLI bundle config-remote-sync

title "Plan after sync"
trace $CLI bundle plan
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"]
# The principals in this test do not exist in a real workspace.
Cloud = false
5 changes: 3 additions & 2 deletions bundle/configsync/diff.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,11 +157,12 @@ func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan

if entry.Changes != nil {
for path, changeDesc := range entry.Changes {
if changeDesc.Action == deployplan.Skip && changeDesc.Reason != deployplan.ReasonRemoteAddition {
fullPath := resourceKey + "." + path
if changeDesc.Action == deployplan.Skip && changeDesc.Reason != deployplan.ReasonRemoteAddition &&
!keepSkippedRunAsChange(fullPath, changeDesc) {
continue
}

fullPath := resourceKey + "." + path
change, err := convertChangeDesc(fullPath, changeDesc)
if err != nil {
return nil, fmt.Errorf("failed to compute config change for path %s: %w", path, err)
Expand Down
15 changes: 15 additions & 0 deletions bundle/configsync/runas.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package configsync

import "github.com/databricks/cli/bundle/deployplan"

// The plan skips a job's run_as keys that the config lacks as backend defaults.
// run_as holds a single principal key, so when the remote switches kind (for
// example user_name to service_principal_name) the new key is the edit itself,
// and dropping it while removing the old key leaves "run_as: {}".
const jobRunAsChildPattern = "resources.jobs.*.run_as.*"

// keepSkippedRunAsChange reports whether a run_as change the plan skipped as a
// backend default should still be considered for syncing.
func keepSkippedRunAsChange(path string, cd *deployplan.ChangeDesc) bool {
return cd.Reason == deployplan.ReasonBackendDefault && matchPattern(jobRunAsChildPattern, path)
}
33 changes: 33 additions & 0 deletions bundle/configsync/runas_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package configsync

import (
"testing"

"github.com/databricks/cli/bundle/deployplan"
"github.com/stretchr/testify/assert"
)

const testRunAsPath = "resources.jobs.my_job.run_as"

func TestKeepSkippedRunAsChange(t *testing.T) {
backendDefault := &deployplan.ChangeDesc{Action: deployplan.Skip, Reason: deployplan.ReasonBackendDefault}
tests := []struct {
name string
path string
cd *deployplan.ChangeDesc
want bool
}{
{"service_principal_name backend default", testRunAsPath + ".service_principal_name", backendDefault, true},
{"user_name backend default", testRunAsPath + ".user_name", backendDefault, true},
{"group_name backend default", testRunAsPath + ".group_name", backendDefault, true},
{"whole run_as backend default", testRunAsPath, backendDefault, false},
{"run_as key already matches the remote", testRunAsPath + ".user_name", &deployplan.ChangeDesc{Action: deployplan.Skip, Reason: deployplan.ReasonRemoteAlreadySet}, false},
{"other job field", "resources.jobs.my_job.tasks[task_key='main'].new_cluster.enable_elastic_disk", backendDefault, false},
{"pipeline run_as key", "resources.pipelines.my_pipeline.run_as.user_name", backendDefault, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, keepSkippedRunAsChange(tt.path, tt.cd))
})
}
}
Loading