diff --git a/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl new file mode 100644 index 00000000000..c655e36c9c2 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl @@ -0,0 +1,31 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + jobs: + user_to_user: + name: user-to-user-$UNIQUE_NAME + run_as: + user_name: first-user@example.com + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + + user_to_creator: + name: user-to-creator-$UNIQUE_NAME + run_as: + user_name: first-user@example.com + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + + user_to_sp: + name: user-to-sp-$UNIQUE_NAME + run_as: + user_name: first-user@example.com + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook diff --git a/acceptance/bundle/config-remote-sync/job_run_as/out.test.toml b/acceptance/bundle/config-remote-sync/job_run_as/out.test.toml new file mode 100644 index 00000000000..8c2cb31de53 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/job_run_as/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +GOOS.windows = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/job_run_as/output.txt b/acceptance/bundle/config-remote-sync/job_run_as/output.txt new file mode 100644 index 00000000000..424cac25200 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/job_run_as/output.txt @@ -0,0 +1,73 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created jobs.user_to_creator +Created jobs.user_to_sp +Created jobs.user_to_user +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Change run_as remotely + +=== Detect and save changes + +>>> [CLI] bundle config-remote-sync --save +Detected changes in 3 resource(s): + +Resource: resources.jobs.user_to_creator + run_as.user_name: replace + +Resource: resources.jobs.user_to_sp + run_as.service_principal_name: add + run_as.user_name: remove + +Resource: resources.jobs.user_to_user + run_as.user_name: replace + + + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -7,5 +7,5 @@ + name: user-to-user-[UNIQUE_NAME] + run_as: +- user_name: first-user@example.com ++ user_name: second-user@example.com + tasks: + - task_key: main +@@ -16,5 +16,5 @@ + name: user-to-creator-[UNIQUE_NAME] + run_as: +- user_name: first-user@example.com ++ user_name: [USERNAME] + tasks: + - task_key: main +@@ -25,5 +25,5 @@ + name: user-to-sp-[UNIQUE_NAME] + run_as: +- user_name: first-user@example.com ++ service_principal_name: [UUID] + tasks: + - task_key: main + +=== Second sync is a no-op + +>>> [CLI] bundle config-remote-sync +No changes detected. + + +=== Plan after sync + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.user_to_creator + delete resources.jobs.user_to_sp + delete resources.jobs.user_to_user + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/config-remote-sync/job_run_as/script b/acceptance/bundle/config-remote-sync/job_run_as/script new file mode 100644 index 00000000000..5beb36780a5 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/job_run_as/script @@ -0,0 +1,31 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy + +title "Change run_as remotely" +edit_resource.py jobs "$(read_id.py user_to_user)" <<'PY' +r["run_as"] = {"user_name": "second-user@example.com"} +PY +edit_resource.py jobs "$(read_id.py user_to_creator)" <>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created jobs.inherited_to_sp +Created jobs.inherited_to_user +Created jobs.inherited_unchanged +Created jobs.target_override_to_group +Created jobs.variable_to_user +Files: 2 uploaded, 0 deleted +Resources: 5 created, 0 changed, 0 deleted, 0 unchanged + +=== Change run_as remotely + +=== Detect and save changes + +>>> [CLI] bundle config-remote-sync --save +Detected changes in 4 resource(s): + +Resource: resources.jobs.inherited_to_sp + run_as.service_principal_name: add + run_as.user_name: remove + +Resource: resources.jobs.inherited_to_user + run_as.user_name: add + +Resource: resources.jobs.target_override_to_group + run_as.group_name: add + run_as.user_name: remove + +Resource: resources.jobs.variable_to_user + run_as.user_name: replace + + + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -26,4 +26,6 @@ + notebook_path: /Users/{{workspace_user_name}}/notebook + ++ run_as: ++ user_name: second-user@example.com + inherited_to_sp: + name: inherited-to-sp-[UNIQUE_NAME] +@@ -33,8 +35,10 @@ + notebook_path: /Users/{{workspace_user_name}}/notebook + ++ run_as: ++ service_principal_name: [UUID] + variable_to_user: + name: variable-to-user-[UNIQUE_NAME] + run_as: +- user_name: ${var.runner} ++ user_name: second-user@example.com + tasks: + - task_key: main +@@ -55,3 +59,3 @@ + target_override_to_group: + run_as: +- user_name: target-user@example.com ++ group_name: data-engineers + +=== Second sync is a no-op + +>>> [CLI] bundle config-remote-sync +No changes detected. + + +=== Plan after sync + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.inherited_to_sp + delete resources.jobs.inherited_to_user + delete resources.jobs.inherited_unchanged + delete resources.jobs.target_override_to_group + delete resources.jobs.variable_to_user + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 5 deleted diff --git a/acceptance/bundle/config-remote-sync/job_run_as_sources/script b/acceptance/bundle/config-remote-sync/job_run_as_sources/script new file mode 100644 index 00000000000..8218d0e4ceb --- /dev/null +++ b/acceptance/bundle/config-remote-sync/job_run_as_sources/script @@ -0,0 +1,34 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy + +title "Change run_as remotely" +edit_resource.py jobs "$(read_id.py inherited_to_user)" <<'PY' +r["run_as"] = {"user_name": "second-user@example.com"} +PY +edit_resource.py jobs "$(read_id.py inherited_to_sp)" <<'PY' +r["run_as"] = {"service_principal_name": "11111111-2222-3333-4444-555555555555"} +PY +edit_resource.py jobs "$(read_id.py variable_to_user)" <<'PY' +r["run_as"] = {"user_name": "second-user@example.com"} +PY +edit_resource.py jobs "$(read_id.py target_override_to_group)" <<'PY' +r["run_as"] = {"group_name": "data-engineers"} +PY + +title "Detect and save changes" +cp databricks.yml databricks.yml.backup +trace $CLI bundle config-remote-sync --save +trace diff.py databricks.yml.backup databricks.yml +rm databricks.yml.backup + +title "Second sync is a no-op" +trace $CLI bundle config-remote-sync + +title "Plan after sync" +trace $CLI bundle plan diff --git a/acceptance/bundle/config-remote-sync/job_run_as_sources/test.toml b/acceptance/bundle/config-remote-sync/job_run_as_sources/test.toml new file mode 100644 index 00000000000..74d09e0bb55 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/job_run_as_sources/test.toml @@ -0,0 +1,3 @@ +Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"] +# The principals in this test do not exist in a real workspace. +Cloud = false diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index e42f0a1d83a..37c296e1ffc 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -157,11 +157,12 @@ func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan if entry.Changes != nil { for path, changeDesc := range entry.Changes { - if changeDesc.Action == deployplan.Skip && changeDesc.Reason != deployplan.ReasonRemoteAddition { + fullPath := resourceKey + "." + path + if changeDesc.Action == deployplan.Skip && changeDesc.Reason != deployplan.ReasonRemoteAddition && + !keepSkippedRunAsChange(fullPath, changeDesc) { continue } - fullPath := resourceKey + "." + path change, err := convertChangeDesc(fullPath, changeDesc) if err != nil { return nil, fmt.Errorf("failed to compute config change for path %s: %w", path, err) diff --git a/bundle/configsync/runas.go b/bundle/configsync/runas.go new file mode 100644 index 00000000000..3faea944f41 --- /dev/null +++ b/bundle/configsync/runas.go @@ -0,0 +1,15 @@ +package configsync + +import "github.com/databricks/cli/bundle/deployplan" + +// The plan skips a job's run_as keys that the config lacks as backend defaults. +// run_as holds a single principal key, so when the remote switches kind (for +// example user_name to service_principal_name) the new key is the edit itself, +// and dropping it while removing the old key leaves "run_as: {}". +const jobRunAsChildPattern = "resources.jobs.*.run_as.*" + +// keepSkippedRunAsChange reports whether a run_as change the plan skipped as a +// backend default should still be considered for syncing. +func keepSkippedRunAsChange(path string, cd *deployplan.ChangeDesc) bool { + return cd.Reason == deployplan.ReasonBackendDefault && matchPattern(jobRunAsChildPattern, path) +} diff --git a/bundle/configsync/runas_test.go b/bundle/configsync/runas_test.go new file mode 100644 index 00000000000..3c1cfbf0f61 --- /dev/null +++ b/bundle/configsync/runas_test.go @@ -0,0 +1,33 @@ +package configsync + +import ( + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/stretchr/testify/assert" +) + +const testRunAsPath = "resources.jobs.my_job.run_as" + +func TestKeepSkippedRunAsChange(t *testing.T) { + backendDefault := &deployplan.ChangeDesc{Action: deployplan.Skip, Reason: deployplan.ReasonBackendDefault} + tests := []struct { + name string + path string + cd *deployplan.ChangeDesc + want bool + }{ + {"service_principal_name backend default", testRunAsPath + ".service_principal_name", backendDefault, true}, + {"user_name backend default", testRunAsPath + ".user_name", backendDefault, true}, + {"group_name backend default", testRunAsPath + ".group_name", backendDefault, true}, + {"whole run_as backend default", testRunAsPath, backendDefault, false}, + {"run_as key already matches the remote", testRunAsPath + ".user_name", &deployplan.ChangeDesc{Action: deployplan.Skip, Reason: deployplan.ReasonRemoteAlreadySet}, false}, + {"other job field", "resources.jobs.my_job.tasks[task_key='main'].new_cluster.enable_elastic_disk", backendDefault, false}, + {"pipeline run_as key", "resources.pipelines.my_pipeline.run_as.user_name", backendDefault, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, keepSkippedRunAsChange(tt.path, tt.cd)) + }) + } +}