From 122f1e9a64f333c65fd0f534793f5ce385c70b20 Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Thu, 12 Nov 2026 18:49:16 +0000 Subject: [PATCH 1/5] Do not report single-node cluster settings supplied by the backend as drift When a cluster spec sets is_single_node: true with kind: CLASSIC_PREVIEW, the backend fills in spark_conf (spark.databricks.cluster.profile=singleNode, spark.master=local[*, N]) and custom_tags.ResourceClass=SingleNode, and they read back next to the flag. The direct engine planner compared that against a config that never declared them and reported `spark_conf: {...} -> ` and the same for custom_tags on every `bundle plan`, for standalone clusters and for job clusters alike. Unlike the cluster-policy case there is no policy to gate on, so the existing ignore_remote_additions rule did not apply. Declare those three keys as backend_defaults (pinned to their known values where the value is fixed) for clusters and for every job cluster-spec location. This only fires when the key is absent from both old and new state, so an explicitly configured spark_conf or custom_tags key still participates in drift detection. A when_set: is_single_node remote-addition gate was rejected because it would hide every out-of-band addition on single-node clusters, not just the three settings the flag generates. Pipeline clusters have no is_single_node field, so nothing changes there. The fake server now expands is_single_node the same way on cluster create and edit and on job cluster specs, so the acceptance tests reproduce the drift. Fixes #7563 Co-authored-by: Isaac --- .nextchanges/bundles/7563.md | 1 + .../clusters/single_node/databricks.yml.tmpl | 30 +++++++ .../clusters/single_node/out.test.toml | 3 + .../resources/clusters/single_node/output.txt | 90 +++++++++++++++++++ .../resources/clusters/single_node/script | 25 ++++++ .../resources/clusters/single_node/test.toml | 12 +++ .../jobs/single_node/databricks.yml.tmpl | 45 ++++++++++ .../resources/jobs/single_node/hello_world.py | 1 + .../resources/jobs/single_node/out.test.toml | 2 + .../resources/jobs/single_node/output.txt | 70 +++++++++++++++ .../bundle/resources/jobs/single_node/script | 15 ++++ .../resources/jobs/single_node/test.toml | 9 ++ bundle/direct/bundle_plan_test.go | 31 +++++++ bundle/direct/dresources/configs/clusters.yml | 13 +++ bundle/direct/dresources/configs/jobs.yml | 20 +++++ libs/testserver/cluster_policies.go | 9 +- libs/testserver/clusters.go | 21 +++++ 17 files changed, 396 insertions(+), 1 deletion(-) create mode 100644 .nextchanges/bundles/7563.md create mode 100644 acceptance/bundle/resources/clusters/single_node/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/clusters/single_node/out.test.toml create mode 100644 acceptance/bundle/resources/clusters/single_node/output.txt create mode 100644 acceptance/bundle/resources/clusters/single_node/script create mode 100644 acceptance/bundle/resources/clusters/single_node/test.toml create mode 100644 acceptance/bundle/resources/jobs/single_node/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/jobs/single_node/hello_world.py create mode 100644 acceptance/bundle/resources/jobs/single_node/out.test.toml create mode 100644 acceptance/bundle/resources/jobs/single_node/output.txt create mode 100644 acceptance/bundle/resources/jobs/single_node/script create mode 100644 acceptance/bundle/resources/jobs/single_node/test.toml diff --git a/.nextchanges/bundles/7563.md b/.nextchanges/bundles/7563.md new file mode 100644 index 00000000000..5000d9d6f10 --- /dev/null +++ b/.nextchanges/bundles/7563.md @@ -0,0 +1 @@ +* Fix `bundle plan` reporting the `spark_conf` and `custom_tags` the backend derives from `is_single_node: true` as drift on standalone and job clusters ([#NNNN](https://github.com/databricks/cli/pull/NNNN)). diff --git a/acceptance/bundle/resources/clusters/single_node/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/single_node/databricks.yml.tmpl new file mode 100644 index 00000000000..99703c93043 --- /dev/null +++ b/acceptance/bundle/resources/clusters/single_node/databricks.yml.tmpl @@ -0,0 +1,30 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + clusters: + # No spark_conf at all: the backend adds the whole map from is_single_node. + implicit: + cluster_name: test-cluster-implicit-$UNIQUE_NAME + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + is_single_node: true + kind: CLASSIC_PREVIEW + data_security_mode: SINGLE_USER + autotermination_minutes: 10 + + # An explicit spark_conf key next to the backend-added ones: it must still be + # compared against the remote. + explicit: + cluster_name: test-cluster-explicit-$UNIQUE_NAME + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + is_single_node: true + kind: CLASSIC_PREVIEW + data_security_mode: SINGLE_USER + autotermination_minutes: 10 + spark_conf: + spark.sql.shuffle.partitions: "8" diff --git a/acceptance/bundle/resources/clusters/single_node/out.test.toml b/acceptance/bundle/resources/clusters/single_node/out.test.toml new file mode 100644 index 00000000000..8bfa768b35b --- /dev/null +++ b/acceptance/bundle/resources/clusters/single_node/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +CloudSlow = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/single_node/output.txt b/acceptance/bundle/resources/clusters/single_node/output.txt new file mode 100644 index 00000000000..63dd3ee869d --- /dev/null +++ b/acceptance/bundle/resources/clusters/single_node/output.txt @@ -0,0 +1,90 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created clusters.explicit +Created clusters.implicit +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Settings the backend derives from is_single_node are not drift + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== ...classified backend_default + +>>> [CLI] bundle plan -o json +{ + "resources.clusters.explicit": { + "custom_tags": { + "action": "skip", + "reason": "backend_default", + "remote": { + "ResourceClass": "SingleNode" + } + }, + "spark_conf['spark.databricks.cluster.profile']": { + "action": "skip", + "reason": "backend_default", + "remote": "singleNode" + }, + "spark_conf['spark.master']": { + "action": "skip", + "reason": "backend_default", + "remote": "local[*, 4]" + } + }, + "resources.clusters.implicit": { + "custom_tags": { + "action": "skip", + "reason": "backend_default", + "remote": { + "ResourceClass": "SingleNode" + } + }, + "spark_conf": { + "action": "skip", + "reason": "backend_default", + "remote": { + "spark.databricks.cluster.profile": "singleNode", + "spark.master": "local[*, 4]" + } + } + } +} + +=== An explicitly configured spark_conf key changed out-of-band is still drift + +>>> [CLI] bundle plan +update clusters.explicit + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle plan -o json +{ + "spark_conf['spark.databricks.cluster.profile']": { + "action": "skip", + "reason": "backend_default", + "remote": "singleNode" + }, + "spark_conf['spark.master']": { + "action": "skip", + "reason": "backend_default", + "remote": "local[*, 4]" + }, + "spark_conf['spark.sql.shuffle.partitions']": { + "action": "update", + "new": "8", + "old": "8", + "remote": "16" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.explicit + delete resources.clusters.implicit + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/clusters/single_node/script b/acceptance/bundle/resources/clusters/single_node/script new file mode 100644 index 00000000000..d18332dce52 --- /dev/null +++ b/acceptance/bundle/resources/clusters/single_node/script @@ -0,0 +1,25 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy + +title "Settings the backend derives from is_single_node are not drift" +trace $CLI bundle plan + +title "...classified backend_default" +trace $CLI bundle plan -o json | jq -S '.plan | with_entries(.value |= (.changes // {} + | with_entries(select(.key | test("spark_conf|custom_tags")))))' + +title "An explicitly configured spark_conf key changed out-of-band is still drift" +explicit_id=$(read_id.py explicit) +$CLI clusters get "$explicit_id" | jq '{cluster_id, cluster_name, spark_version, node_type_id, + is_single_node, kind, data_security_mode, autotermination_minutes, + spark_conf: (.spark_conf + {"spark.sql.shuffle.partitions": "16"})}' > edit.json +trace $CLI clusters edit --json @edit.json &> LOG.edit +trace $CLI bundle plan +trace $CLI bundle plan -o json | jq -S '.plan["resources.clusters.explicit"].changes + | with_entries(select(.key | test("spark_conf")))' diff --git a/acceptance/bundle/resources/clusters/single_node/test.toml b/acceptance/bundle/resources/clusters/single_node/test.toml new file mode 100644 index 00000000000..71cc13906f7 --- /dev/null +++ b/acceptance/bundle/resources/clusters/single_node/test.toml @@ -0,0 +1,12 @@ +# The standalone clusters resource boots a real cluster at deploy, which is slow and has hit +# capacity limits on cloud; CloudSlow keeps it out of the per-PR run (it still runs locally and +# in the full cloud sweep). Output is projected to spark_conf/custom_tags so it is cloud-portable. +Cloud = true +CloudSlow = true + +RecordRequests = false + +Ignore = [ + "databricks.yml", + "edit.json", +] diff --git a/acceptance/bundle/resources/jobs/single_node/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/single_node/databricks.yml.tmpl new file mode 100644 index 00000000000..1e9301f9f72 --- /dev/null +++ b/acceptance/bundle/resources/jobs/single_node/databricks.yml.tmpl @@ -0,0 +1,45 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + jobs: + my_job: + name: test-job-$UNIQUE_NAME + job_clusters: + - job_cluster_key: job_cluster + new_cluster: + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + is_single_node: true + kind: CLASSIC_PREVIEW + data_security_mode: SINGLE_USER + tasks: + - task_key: on_job_cluster + job_cluster_key: job_cluster + spark_python_task: + python_file: ./hello_world.py + - task_key: task_cluster + new_cluster: + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + is_single_node: true + kind: CLASSIC_PREVIEW + data_security_mode: SINGLE_USER + spark_python_task: + python_file: ./hello_world.py + - task_key: for_each + for_each_task: + inputs: "[1]" + task: + task_key: for_each_cluster + new_cluster: + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + is_single_node: true + kind: CLASSIC_PREVIEW + data_security_mode: SINGLE_USER + spark_python_task: + python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/jobs/single_node/hello_world.py b/acceptance/bundle/resources/jobs/single_node/hello_world.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/acceptance/bundle/resources/jobs/single_node/hello_world.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/resources/jobs/single_node/out.test.toml b/acceptance/bundle/resources/jobs/single_node/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/jobs/single_node/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/single_node/output.txt b/acceptance/bundle/resources/jobs/single_node/output.txt new file mode 100644 index 00000000000..ea1d50825a4 --- /dev/null +++ b/acceptance/bundle/resources/jobs/single_node/output.txt @@ -0,0 +1,70 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created jobs.my_job +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Settings the backend derives from is_single_node are not drift at any cluster-spec location + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Each is classified backend_default + +>>> [CLI] bundle plan -o json +{ + "job_clusters[job_cluster_key='job_cluster'].new_cluster.custom_tags": { + "action": "skip", + "reason": "backend_default", + "remote": { + "ResourceClass": "SingleNode" + } + }, + "job_clusters[job_cluster_key='job_cluster'].new_cluster.spark_conf": { + "action": "skip", + "reason": "backend_default", + "remote": { + "spark.databricks.cluster.profile": "singleNode", + "spark.master": "local[*, 4]" + } + }, + "tasks[task_key='for_each'].for_each_task.task.new_cluster.custom_tags": { + "action": "skip", + "reason": "backend_default", + "remote": { + "ResourceClass": "SingleNode" + } + }, + "tasks[task_key='for_each'].for_each_task.task.new_cluster.spark_conf": { + "action": "skip", + "reason": "backend_default", + "remote": { + "spark.databricks.cluster.profile": "singleNode", + "spark.master": "local[*, 4]" + } + }, + "tasks[task_key='task_cluster'].new_cluster.custom_tags": { + "action": "skip", + "reason": "backend_default", + "remote": { + "ResourceClass": "SingleNode" + } + }, + "tasks[task_key='task_cluster'].new_cluster.spark_conf": { + "action": "skip", + "reason": "backend_default", + "remote": { + "spark.databricks.cluster.profile": "singleNode", + "spark.master": "local[*, 4]" + } + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/jobs/single_node/script b/acceptance/bundle/resources/jobs/single_node/script new file mode 100644 index 00000000000..3b0171cae5d --- /dev/null +++ b/acceptance/bundle/resources/jobs/single_node/script @@ -0,0 +1,15 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy + +title "Settings the backend derives from is_single_node are not drift at any cluster-spec location" +trace $CLI bundle plan + +title "Each is classified backend_default" +trace $CLI bundle plan -o json | jq -S '.plan["resources.jobs.my_job"].changes + | with_entries(select(.key | test("spark_conf|custom_tags")))' diff --git a/acceptance/bundle/resources/jobs/single_node/test.toml b/acceptance/bundle/resources/jobs/single_node/test.toml new file mode 100644 index 00000000000..834e9fcaf63 --- /dev/null +++ b/acceptance/bundle/resources/jobs/single_node/test.toml @@ -0,0 +1,9 @@ +# The drift-suppression under test is a direct-engine feature (bundle/direct). Cloud = true adds +# a real-workspace run; the test also runs locally against the testserver. +Cloud = true + +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 2d051dc21b9..72f572fbc44 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -795,3 +795,34 @@ func TestLadderBackendDefaultBeforeRemoteAddition(t *testing.T) { assert.Equal(t, deployplan.Skip, changes["enable_elastic_disk"].Action) assert.Equal(t, deployplan.ReasonBackendDefault, changes["enable_elastic_disk"].Reason) } + +// TestClustersSingleNodeBackendDefaults pins the clusters.yml rules for the settings the +// backend derives from is_single_node (#7563). The whole-map shape is what a config without +// spark_conf produces; the per-key shape is what a config with its own key produces, and an +// explicit key that drifted must stay an update. +func TestClustersSingleNodeBackendDefaults(t *testing.T) { + adapter, err := dresources.NewAdapter(dresources.SupportedResources["clusters"], "clusters", nil) + require.NoError(t, err) + + newState := &dresources.ClusterState{IsSingleNode: true, Kind: compute.KindClassicPreview} + changes := deployplan.Changes{ + "spark_conf": &deployplan.ChangeDesc{Remote: map[string]string{ + "spark.databricks.cluster.profile": "singleNode", + "spark.master": "local[*, 4]", + }}, + "custom_tags": &deployplan.ChangeDesc{Remote: map[string]string{"ResourceClass": "SingleNode"}}, + "spark_conf['spark.databricks.cluster.profile']": &deployplan.ChangeDesc{Remote: "singleNode"}, + "spark_conf['spark.master']": &deployplan.ChangeDesc{Remote: "local[*, 8]"}, + "spark_conf['spark.sql.shuffle.partitions']": &deployplan.ChangeDesc{Old: "8", New: "8", Remote: "16"}, + // A ResourceClass other than SingleNode is not something is_single_node produces. + "custom_tags['ResourceClass']": &deployplan.ChangeDesc{Remote: "Serverless"}, + } + require.NoError(t, addPerFieldActions(t.Context(), adapter, changes, newState, nil)) + + for _, path := range []string{"spark_conf", "custom_tags", "spark_conf['spark.databricks.cluster.profile']", "spark_conf['spark.master']"} { + assert.Equal(t, deployplan.Skip, changes[path].Action, path) + assert.Equal(t, deployplan.ReasonBackendDefault, changes[path].Reason, path) + } + assert.Equal(t, deployplan.Update, changes["spark_conf['spark.sql.shuffle.partitions']"].Action) + assert.Equal(t, deployplan.Update, changes["custom_tags['ResourceClass']"].Action) +} diff --git a/bundle/direct/dresources/configs/clusters.yml b/bundle/direct/dresources/configs/clusters.yml index b19d9c0483c..da471f866ee 100644 --- a/bundle/direct/dresources/configs/clusters.yml +++ b/bundle/direct/dresources/configs/clusters.yml @@ -51,6 +51,19 @@ backend_defaults: # See https://github.com/databricks/cli/issues/4418 - field: single_user_name + # is_single_node on a kind: CLASSIC_PREVIEW cluster makes the backend fill in the legacy + # single-node settings, which read back next to the flag. num_workers is forced to 0 as + # well, but that is the zero value and never diffs. Explicitly configured spark_conf and + # custom_tags keys are unaffected and still participate in drift detection. + # https://docs.databricks.com/api/workspace/clusters/create#is_single_node + # https://github.com/databricks/cli/issues/7563 + - field: spark_conf['spark.databricks.cluster.profile'] + values: ["singleNode"] + # The core count in local[*, N] follows the node type, so the value cannot be pinned. + - field: spark_conf['spark.master'] + - field: custom_tags['ResourceClass'] + values: ["SingleNode"] + # We have custom handler for this in cluster.go # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/clusters/resource_cluster.go#L109-L118 # DataSecurityModeDiffSuppressFunc: suppress when old != "" && new == "" diff --git a/bundle/direct/dresources/configs/jobs.yml b/bundle/direct/dresources/configs/jobs.yml index c1a9b5a5790..7200ce8c1d9 100644 --- a/bundle/direct/dresources/configs/jobs.yml +++ b/bundle/direct/dresources/configs/jobs.yml @@ -107,6 +107,26 @@ backend_defaults: - field: tasks[*].for_each_task.task.new_cluster.driver_instance_pool_id - field: job_clusters[*].new_cluster.driver_instance_pool_id + # Same as clusters.spark_conf/custom_tags: is_single_node on a kind: CLASSIC_PREVIEW + # cluster spec makes the backend fill in the legacy single-node settings. + # https://docs.databricks.com/api/workspace/clusters/create#is_single_node + # https://github.com/databricks/cli/issues/7563 + - field: tasks[*].new_cluster.spark_conf['spark.databricks.cluster.profile'] + values: ["singleNode"] + - field: tasks[*].for_each_task.task.new_cluster.spark_conf['spark.databricks.cluster.profile'] + values: ["singleNode"] + - field: job_clusters[*].new_cluster.spark_conf['spark.databricks.cluster.profile'] + values: ["singleNode"] + - field: tasks[*].new_cluster.spark_conf['spark.master'] + - field: tasks[*].for_each_task.task.new_cluster.spark_conf['spark.master'] + - field: job_clusters[*].new_cluster.spark_conf['spark.master'] + - field: tasks[*].new_cluster.custom_tags['ResourceClass'] + values: ["SingleNode"] + - field: tasks[*].for_each_task.task.new_cluster.custom_tags['ResourceClass'] + values: ["SingleNode"] + - field: job_clusters[*].new_cluster.custom_tags['ResourceClass'] + values: ["SingleNode"] + # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/jobs/resource_job.go#L531 # s.SchemaPath("format").SetComputed() - field: format diff --git a/libs/testserver/cluster_policies.go b/libs/testserver/cluster_policies.go index 7351e357d11..06f41c7fc27 100644 --- a/libs/testserver/cluster_policies.go +++ b/libs/testserver/cluster_policies.go @@ -260,9 +260,16 @@ func (s *FakeWorkspace) applyJobClusterPolicies(settings *jobs.JobSettings) stri return "" } +// applyClusterSpecPolicy applies the attached policy and the server-side single-node +// expansion to one job cluster spec; the Jobs API stores the same expanded spec as +// the Clusters API does. func (s *FakeWorkspace) applyClusterSpecPolicy(spec *compute.ClusterSpec) string { if spec == nil { return "" } - return s.applyClusterPolicy(spec, spec.PolicyId, spec.ApplyPolicyDefaultValues) + if msg := s.applyClusterPolicy(spec, spec.PolicyId, spec.ApplyPolicyDefaultValues); msg != "" { + return msg + } + applySingleNodeDefaults(spec.IsSingleNode, spec.Kind, &spec.SparkConf, &spec.CustomTags) + return "" } diff --git a/libs/testserver/clusters.go b/libs/testserver/clusters.go index e0a046031e6..0fa88af9bf9 100644 --- a/libs/testserver/clusters.go +++ b/libs/testserver/clusters.go @@ -179,6 +179,27 @@ func clusterFixUps(cluster *compute.ClusterDetails) { if cluster.DriverNodeTypeId == "" && cluster.NodeTypeId != "" { cluster.DriverNodeTypeId = cluster.NodeTypeId } + + applySingleNodeDefaults(cluster.IsSingleNode, cluster.Kind, &cluster.SparkConf, &cluster.CustomTags) +} + +// applySingleNodeDefaults mirrors the backend expanding is_single_node on a CLASSIC_PREVIEW +// cluster spec into the legacy single-node settings, which then read back next to the flag. +// num_workers is also forced to 0 but that is the Go zero value already. +// https://docs.databricks.com/api/workspace/clusters/create#is_single_node +func applySingleNodeDefaults(isSingleNode bool, kind compute.Kind, sparkConf, customTags *map[string]string) { + if !isSingleNode || kind != compute.KindClassicPreview { + return + } + if *sparkConf == nil { + *sparkConf = map[string]string{} + } + (*sparkConf)["spark.databricks.cluster.profile"] = "singleNode" + (*sparkConf)["spark.master"] = "local[*, 4]" + if *customTags == nil { + *customTags = map[string]string{} + } + (*customTags)["ResourceClass"] = "SingleNode" } func (s *FakeWorkspace) ClustersGet(req Request, clusterId string) any { From 808a564ebb5cd1a8af84bc2578cf86507a076018 Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Thu, 12 Nov 2026 21:38:39 +0000 Subject: [PATCH 2/5] Link changelog fragment to PR #7577 --- .nextchanges/bundles/7563.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/7563.md b/.nextchanges/bundles/7563.md index 5000d9d6f10..50afd973387 100644 --- a/.nextchanges/bundles/7563.md +++ b/.nextchanges/bundles/7563.md @@ -1 +1 @@ -* Fix `bundle plan` reporting the `spark_conf` and `custom_tags` the backend derives from `is_single_node: true` as drift on standalone and job clusters ([#NNNN](https://github.com/databricks/cli/pull/NNNN)). +* Fix `bundle plan` reporting the `spark_conf` and `custom_tags` the backend derives from `is_single_node: true` as drift on standalone and job clusters ([#7577](https://github.com/databricks/cli/pull/7577)). From 01d0d61ec7993a7ab36eef5e367049a7a7280cb1 Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Thu, 12 Nov 2026 23:39:55 +0000 Subject: [PATCH 3/5] Put the changelog fragment's period before the PR link --- .nextchanges/bundles/7563.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/7563.md b/.nextchanges/bundles/7563.md index 50afd973387..e79cbd0baaf 100644 --- a/.nextchanges/bundles/7563.md +++ b/.nextchanges/bundles/7563.md @@ -1 +1 @@ -* Fix `bundle plan` reporting the `spark_conf` and `custom_tags` the backend derives from `is_single_node: true` as drift on standalone and job clusters ([#7577](https://github.com/databricks/cli/pull/7577)). +* Fix `bundle plan` reporting the `spark_conf` and `custom_tags` the backend derives from `is_single_node: true` as drift on standalone and job clusters. ([#7577](https://github.com/databricks/cli/pull/7577)) From ab0a36194f3f58e3f085a2e9b561f399acb7a957 Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Fri, 13 Nov 2026 11:05:58 +0000 Subject: [PATCH 4/5] Only expand is_single_node in the fake Clusters API, not on job cluster specs Against a real workspace the Jobs API stores new_cluster specs as authored, so the backend_defaults rules for job clusters never matched and the jobs/single_node cloud run failed with an empty plan. Drop the rules, the fake expansion on job cluster specs, and the test that depended on them. Co-authored-by: Isaac --- .nextchanges/bundles/7563.md | 2 +- .../jobs/single_node/databricks.yml.tmpl | 45 ------------ .../resources/jobs/single_node/hello_world.py | 1 - .../resources/jobs/single_node/out.test.toml | 2 - .../resources/jobs/single_node/output.txt | 70 ------------------- .../bundle/resources/jobs/single_node/script | 15 ---- .../resources/jobs/single_node/test.toml | 9 --- bundle/direct/dresources/configs/jobs.yml | 20 ------ libs/testserver/cluster_policies.go | 9 +-- libs/testserver/clusters.go | 27 +++---- 10 files changed, 16 insertions(+), 184 deletions(-) delete mode 100644 acceptance/bundle/resources/jobs/single_node/databricks.yml.tmpl delete mode 100644 acceptance/bundle/resources/jobs/single_node/hello_world.py delete mode 100644 acceptance/bundle/resources/jobs/single_node/out.test.toml delete mode 100644 acceptance/bundle/resources/jobs/single_node/output.txt delete mode 100644 acceptance/bundle/resources/jobs/single_node/script delete mode 100644 acceptance/bundle/resources/jobs/single_node/test.toml diff --git a/.nextchanges/bundles/7563.md b/.nextchanges/bundles/7563.md index e79cbd0baaf..067854f1a90 100644 --- a/.nextchanges/bundles/7563.md +++ b/.nextchanges/bundles/7563.md @@ -1 +1 @@ -* Fix `bundle plan` reporting the `spark_conf` and `custom_tags` the backend derives from `is_single_node: true` as drift on standalone and job clusters. ([#7577](https://github.com/databricks/cli/pull/7577)) +* Fix `bundle plan` reporting the `spark_conf` and `custom_tags` the backend derives from `is_single_node: true` as drift on clusters. ([#7577](https://github.com/databricks/cli/pull/7577)) diff --git a/acceptance/bundle/resources/jobs/single_node/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/single_node/databricks.yml.tmpl deleted file mode 100644 index 1e9301f9f72..00000000000 --- a/acceptance/bundle/resources/jobs/single_node/databricks.yml.tmpl +++ /dev/null @@ -1,45 +0,0 @@ -bundle: - name: test-bundle-$UNIQUE_NAME - -workspace: - root_path: ~/.bundle/$UNIQUE_NAME - -resources: - jobs: - my_job: - name: test-job-$UNIQUE_NAME - job_clusters: - - job_cluster_key: job_cluster - new_cluster: - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - is_single_node: true - kind: CLASSIC_PREVIEW - data_security_mode: SINGLE_USER - tasks: - - task_key: on_job_cluster - job_cluster_key: job_cluster - spark_python_task: - python_file: ./hello_world.py - - task_key: task_cluster - new_cluster: - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - is_single_node: true - kind: CLASSIC_PREVIEW - data_security_mode: SINGLE_USER - spark_python_task: - python_file: ./hello_world.py - - task_key: for_each - for_each_task: - inputs: "[1]" - task: - task_key: for_each_cluster - new_cluster: - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - is_single_node: true - kind: CLASSIC_PREVIEW - data_security_mode: SINGLE_USER - spark_python_task: - python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/jobs/single_node/hello_world.py b/acceptance/bundle/resources/jobs/single_node/hello_world.py deleted file mode 100644 index 11b15b1a458..00000000000 --- a/acceptance/bundle/resources/jobs/single_node/hello_world.py +++ /dev/null @@ -1 +0,0 @@ -print("hello") diff --git a/acceptance/bundle/resources/jobs/single_node/out.test.toml b/acceptance/bundle/resources/jobs/single_node/out.test.toml deleted file mode 100644 index 0f167effef1..00000000000 --- a/acceptance/bundle/resources/jobs/single_node/out.test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Cloud = true -EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/single_node/output.txt b/acceptance/bundle/resources/jobs/single_node/output.txt deleted file mode 100644 index ea1d50825a4..00000000000 --- a/acceptance/bundle/resources/jobs/single_node/output.txt +++ /dev/null @@ -1,70 +0,0 @@ - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... -Created jobs.my_job -Files: 3 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - -=== Settings the backend derives from is_single_node are not drift at any cluster-spec location - ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged - -=== Each is classified backend_default - ->>> [CLI] bundle plan -o json -{ - "job_clusters[job_cluster_key='job_cluster'].new_cluster.custom_tags": { - "action": "skip", - "reason": "backend_default", - "remote": { - "ResourceClass": "SingleNode" - } - }, - "job_clusters[job_cluster_key='job_cluster'].new_cluster.spark_conf": { - "action": "skip", - "reason": "backend_default", - "remote": { - "spark.databricks.cluster.profile": "singleNode", - "spark.master": "local[*, 4]" - } - }, - "tasks[task_key='for_each'].for_each_task.task.new_cluster.custom_tags": { - "action": "skip", - "reason": "backend_default", - "remote": { - "ResourceClass": "SingleNode" - } - }, - "tasks[task_key='for_each'].for_each_task.task.new_cluster.spark_conf": { - "action": "skip", - "reason": "backend_default", - "remote": { - "spark.databricks.cluster.profile": "singleNode", - "spark.master": "local[*, 4]" - } - }, - "tasks[task_key='task_cluster'].new_cluster.custom_tags": { - "action": "skip", - "reason": "backend_default", - "remote": { - "ResourceClass": "SingleNode" - } - }, - "tasks[task_key='task_cluster'].new_cluster.spark_conf": { - "action": "skip", - "reason": "backend_default", - "remote": { - "spark.databricks.cluster.profile": "singleNode", - "spark.master": "local[*, 4]" - } - } -} - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.my_job - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] - -Destroy: 1 deleted diff --git a/acceptance/bundle/resources/jobs/single_node/script b/acceptance/bundle/resources/jobs/single_node/script deleted file mode 100644 index 3b0171cae5d..00000000000 --- a/acceptance/bundle/resources/jobs/single_node/script +++ /dev/null @@ -1,15 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -cleanup() { - trace $CLI bundle destroy --auto-approve -} -trap cleanup EXIT - -trace $CLI bundle deploy - -title "Settings the backend derives from is_single_node are not drift at any cluster-spec location" -trace $CLI bundle plan - -title "Each is classified backend_default" -trace $CLI bundle plan -o json | jq -S '.plan["resources.jobs.my_job"].changes - | with_entries(select(.key | test("spark_conf|custom_tags")))' diff --git a/acceptance/bundle/resources/jobs/single_node/test.toml b/acceptance/bundle/resources/jobs/single_node/test.toml deleted file mode 100644 index 834e9fcaf63..00000000000 --- a/acceptance/bundle/resources/jobs/single_node/test.toml +++ /dev/null @@ -1,9 +0,0 @@ -# The drift-suppression under test is a direct-engine feature (bundle/direct). Cloud = true adds -# a real-workspace run; the test also runs locally against the testserver. -Cloud = true - -RecordRequests = false - -Ignore = [ - "databricks.yml", -] diff --git a/bundle/direct/dresources/configs/jobs.yml b/bundle/direct/dresources/configs/jobs.yml index 7200ce8c1d9..c1a9b5a5790 100644 --- a/bundle/direct/dresources/configs/jobs.yml +++ b/bundle/direct/dresources/configs/jobs.yml @@ -107,26 +107,6 @@ backend_defaults: - field: tasks[*].for_each_task.task.new_cluster.driver_instance_pool_id - field: job_clusters[*].new_cluster.driver_instance_pool_id - # Same as clusters.spark_conf/custom_tags: is_single_node on a kind: CLASSIC_PREVIEW - # cluster spec makes the backend fill in the legacy single-node settings. - # https://docs.databricks.com/api/workspace/clusters/create#is_single_node - # https://github.com/databricks/cli/issues/7563 - - field: tasks[*].new_cluster.spark_conf['spark.databricks.cluster.profile'] - values: ["singleNode"] - - field: tasks[*].for_each_task.task.new_cluster.spark_conf['spark.databricks.cluster.profile'] - values: ["singleNode"] - - field: job_clusters[*].new_cluster.spark_conf['spark.databricks.cluster.profile'] - values: ["singleNode"] - - field: tasks[*].new_cluster.spark_conf['spark.master'] - - field: tasks[*].for_each_task.task.new_cluster.spark_conf['spark.master'] - - field: job_clusters[*].new_cluster.spark_conf['spark.master'] - - field: tasks[*].new_cluster.custom_tags['ResourceClass'] - values: ["SingleNode"] - - field: tasks[*].for_each_task.task.new_cluster.custom_tags['ResourceClass'] - values: ["SingleNode"] - - field: job_clusters[*].new_cluster.custom_tags['ResourceClass'] - values: ["SingleNode"] - # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/jobs/resource_job.go#L531 # s.SchemaPath("format").SetComputed() - field: format diff --git a/libs/testserver/cluster_policies.go b/libs/testserver/cluster_policies.go index 06f41c7fc27..7351e357d11 100644 --- a/libs/testserver/cluster_policies.go +++ b/libs/testserver/cluster_policies.go @@ -260,16 +260,9 @@ func (s *FakeWorkspace) applyJobClusterPolicies(settings *jobs.JobSettings) stri return "" } -// applyClusterSpecPolicy applies the attached policy and the server-side single-node -// expansion to one job cluster spec; the Jobs API stores the same expanded spec as -// the Clusters API does. func (s *FakeWorkspace) applyClusterSpecPolicy(spec *compute.ClusterSpec) string { if spec == nil { return "" } - if msg := s.applyClusterPolicy(spec, spec.PolicyId, spec.ApplyPolicyDefaultValues); msg != "" { - return msg - } - applySingleNodeDefaults(spec.IsSingleNode, spec.Kind, &spec.SparkConf, &spec.CustomTags) - return "" + return s.applyClusterPolicy(spec, spec.PolicyId, spec.ApplyPolicyDefaultValues) } diff --git a/libs/testserver/clusters.go b/libs/testserver/clusters.go index 0fa88af9bf9..1451efb9317 100644 --- a/libs/testserver/clusters.go +++ b/libs/testserver/clusters.go @@ -180,26 +180,27 @@ func clusterFixUps(cluster *compute.ClusterDetails) { cluster.DriverNodeTypeId = cluster.NodeTypeId } - applySingleNodeDefaults(cluster.IsSingleNode, cluster.Kind, &cluster.SparkConf, &cluster.CustomTags) + applySingleNodeDefaults(cluster) } -// applySingleNodeDefaults mirrors the backend expanding is_single_node on a CLASSIC_PREVIEW -// cluster spec into the legacy single-node settings, which then read back next to the flag. -// num_workers is also forced to 0 but that is the Go zero value already. +// applySingleNodeDefaults mirrors the Clusters API expanding is_single_node on a CLASSIC_PREVIEW +// cluster into the legacy single-node settings, which then read back next to the flag. +// num_workers is also forced to 0 but that is the Go zero value already. The Jobs API does not +// do this for new_cluster specs: they read back as authored. // https://docs.databricks.com/api/workspace/clusters/create#is_single_node -func applySingleNodeDefaults(isSingleNode bool, kind compute.Kind, sparkConf, customTags *map[string]string) { - if !isSingleNode || kind != compute.KindClassicPreview { +func applySingleNodeDefaults(cluster *compute.ClusterDetails) { + if !cluster.IsSingleNode || cluster.Kind != compute.KindClassicPreview { return } - if *sparkConf == nil { - *sparkConf = map[string]string{} + if cluster.SparkConf == nil { + cluster.SparkConf = map[string]string{} } - (*sparkConf)["spark.databricks.cluster.profile"] = "singleNode" - (*sparkConf)["spark.master"] = "local[*, 4]" - if *customTags == nil { - *customTags = map[string]string{} + cluster.SparkConf["spark.databricks.cluster.profile"] = "singleNode" + cluster.SparkConf["spark.master"] = "local[*, 4]" + if cluster.CustomTags == nil { + cluster.CustomTags = map[string]string{} } - (*customTags)["ResourceClass"] = "SingleNode" + cluster.CustomTags["ResourceClass"] = "SingleNode" } func (s *FakeWorkspace) ClustersGet(req Request, clusterId string) any { From a2ae76d2a4db249dfccd11028421b402f21b806d Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Fri, 13 Nov 2026 11:06:00 +0000 Subject: [PATCH 5/5] Make the clusters single_node acceptance test cloud-portable Normalise the node-type dependent core count in spark.master and stop re-sending the backend-derived spark_conf keys on clusters edit. Co-authored-by: Isaac --- acceptance/bundle/resources/clusters/single_node/output.txt | 6 +++--- acceptance/bundle/resources/clusters/single_node/script | 4 +++- acceptance/bundle/resources/clusters/single_node/test.toml | 5 +++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/clusters/single_node/output.txt b/acceptance/bundle/resources/clusters/single_node/output.txt index 63dd3ee869d..3809f85dfc6 100644 --- a/acceptance/bundle/resources/clusters/single_node/output.txt +++ b/acceptance/bundle/resources/clusters/single_node/output.txt @@ -31,7 +31,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "spark_conf['spark.master']": { "action": "skip", "reason": "backend_default", - "remote": "local[*, 4]" + "remote": "local[*, [CORES]]" } }, "resources.clusters.implicit": { @@ -47,7 +47,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "reason": "backend_default", "remote": { "spark.databricks.cluster.profile": "singleNode", - "spark.master": "local[*, 4]" + "spark.master": "local[*, [CORES]]" } } } @@ -70,7 +70,7 @@ Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged "spark_conf['spark.master']": { "action": "skip", "reason": "backend_default", - "remote": "local[*, 4]" + "remote": "local[*, [CORES]]" }, "spark_conf['spark.sql.shuffle.partitions']": { "action": "update", diff --git a/acceptance/bundle/resources/clusters/single_node/script b/acceptance/bundle/resources/clusters/single_node/script index d18332dce52..4716b302e9d 100644 --- a/acceptance/bundle/resources/clusters/single_node/script +++ b/acceptance/bundle/resources/clusters/single_node/script @@ -16,9 +16,11 @@ trace $CLI bundle plan -o json | jq -S '.plan | with_entries(.value |= (.changes title "An explicitly configured spark_conf key changed out-of-band is still drift" explicit_id=$(read_id.py explicit) +# Re-send only the authored spark_conf keys: the backend-derived ones come back from is_single_node. $CLI clusters get "$explicit_id" | jq '{cluster_id, cluster_name, spark_version, node_type_id, is_single_node, kind, data_security_mode, autotermination_minutes, - spark_conf: (.spark_conf + {"spark.sql.shuffle.partitions": "16"})}' > edit.json + spark_conf: (.spark_conf | del(.["spark.databricks.cluster.profile"], .["spark.master"]) + + {"spark.sql.shuffle.partitions": "16"})}' > edit.json trace $CLI clusters edit --json @edit.json &> LOG.edit trace $CLI bundle plan trace $CLI bundle plan -o json | jq -S '.plan["resources.clusters.explicit"].changes diff --git a/acceptance/bundle/resources/clusters/single_node/test.toml b/acceptance/bundle/resources/clusters/single_node/test.toml index 71cc13906f7..607050ff37f 100644 --- a/acceptance/bundle/resources/clusters/single_node/test.toml +++ b/acceptance/bundle/resources/clusters/single_node/test.toml @@ -10,3 +10,8 @@ Ignore = [ "databricks.yml", "edit.json", ] + +# The core count the backend puts in spark.master follows the node type. +[[Repls]] +Old = 'local\[\*, \d+\]' +New = 'local[*, [CORES]]'