diff --git a/.nextchanges/bundles/7563.md b/.nextchanges/bundles/7563.md new file mode 100644 index 00000000000..067854f1a90 --- /dev/null +++ b/.nextchanges/bundles/7563.md @@ -0,0 +1 @@ +* Fix `bundle plan` reporting the `spark_conf` and `custom_tags` the backend derives from `is_single_node: true` as drift on clusters. ([#7577](https://github.com/databricks/cli/pull/7577)) diff --git a/acceptance/bundle/resources/clusters/single_node/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/single_node/databricks.yml.tmpl new file mode 100644 index 00000000000..99703c93043 --- /dev/null +++ b/acceptance/bundle/resources/clusters/single_node/databricks.yml.tmpl @@ -0,0 +1,30 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + clusters: + # No spark_conf at all: the backend adds the whole map from is_single_node. + implicit: + cluster_name: test-cluster-implicit-$UNIQUE_NAME + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + is_single_node: true + kind: CLASSIC_PREVIEW + data_security_mode: SINGLE_USER + autotermination_minutes: 10 + + # An explicit spark_conf key next to the backend-added ones: it must still be + # compared against the remote. + explicit: + cluster_name: test-cluster-explicit-$UNIQUE_NAME + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + is_single_node: true + kind: CLASSIC_PREVIEW + data_security_mode: SINGLE_USER + autotermination_minutes: 10 + spark_conf: + spark.sql.shuffle.partitions: "8" diff --git a/acceptance/bundle/resources/clusters/single_node/out.test.toml b/acceptance/bundle/resources/clusters/single_node/out.test.toml new file mode 100644 index 00000000000..8bfa768b35b --- /dev/null +++ b/acceptance/bundle/resources/clusters/single_node/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +CloudSlow = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/single_node/output.txt b/acceptance/bundle/resources/clusters/single_node/output.txt new file mode 100644 index 00000000000..3809f85dfc6 --- /dev/null +++ b/acceptance/bundle/resources/clusters/single_node/output.txt @@ -0,0 +1,90 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created clusters.explicit +Created clusters.implicit +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Settings the backend derives from is_single_node are not drift + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== ...classified backend_default + +>>> [CLI] bundle plan -o json +{ + "resources.clusters.explicit": { + "custom_tags": { + "action": "skip", + "reason": "backend_default", + "remote": { + "ResourceClass": "SingleNode" + } + }, + "spark_conf['spark.databricks.cluster.profile']": { + "action": "skip", + "reason": "backend_default", + "remote": "singleNode" + }, + "spark_conf['spark.master']": { + "action": "skip", + "reason": "backend_default", + "remote": "local[*, [CORES]]" + } + }, + "resources.clusters.implicit": { + "custom_tags": { + "action": "skip", + "reason": "backend_default", + "remote": { + "ResourceClass": "SingleNode" + } + }, + "spark_conf": { + "action": "skip", + "reason": "backend_default", + "remote": { + "spark.databricks.cluster.profile": "singleNode", + "spark.master": "local[*, [CORES]]" + } + } + } +} + +=== An explicitly configured spark_conf key changed out-of-band is still drift + +>>> [CLI] bundle plan +update clusters.explicit + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle plan -o json +{ + "spark_conf['spark.databricks.cluster.profile']": { + "action": "skip", + "reason": "backend_default", + "remote": "singleNode" + }, + "spark_conf['spark.master']": { + "action": "skip", + "reason": "backend_default", + "remote": "local[*, [CORES]]" + }, + "spark_conf['spark.sql.shuffle.partitions']": { + "action": "update", + "new": "8", + "old": "8", + "remote": "16" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.explicit + delete resources.clusters.implicit + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/clusters/single_node/script b/acceptance/bundle/resources/clusters/single_node/script new file mode 100644 index 00000000000..4716b302e9d --- /dev/null +++ b/acceptance/bundle/resources/clusters/single_node/script @@ -0,0 +1,27 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy + +title "Settings the backend derives from is_single_node are not drift" +trace $CLI bundle plan + +title "...classified backend_default" +trace $CLI bundle plan -o json | jq -S '.plan | with_entries(.value |= (.changes // {} + | with_entries(select(.key | test("spark_conf|custom_tags")))))' + +title "An explicitly configured spark_conf key changed out-of-band is still drift" +explicit_id=$(read_id.py explicit) +# Re-send only the authored spark_conf keys: the backend-derived ones come back from is_single_node. +$CLI clusters get "$explicit_id" | jq '{cluster_id, cluster_name, spark_version, node_type_id, + is_single_node, kind, data_security_mode, autotermination_minutes, + spark_conf: (.spark_conf | del(.["spark.databricks.cluster.profile"], .["spark.master"]) + + {"spark.sql.shuffle.partitions": "16"})}' > edit.json +trace $CLI clusters edit --json @edit.json &> LOG.edit +trace $CLI bundle plan +trace $CLI bundle plan -o json | jq -S '.plan["resources.clusters.explicit"].changes + | with_entries(select(.key | test("spark_conf")))' diff --git a/acceptance/bundle/resources/clusters/single_node/test.toml b/acceptance/bundle/resources/clusters/single_node/test.toml new file mode 100644 index 00000000000..607050ff37f --- /dev/null +++ b/acceptance/bundle/resources/clusters/single_node/test.toml @@ -0,0 +1,17 @@ +# The standalone clusters resource boots a real cluster at deploy, which is slow and has hit +# capacity limits on cloud; CloudSlow keeps it out of the per-PR run (it still runs locally and +# in the full cloud sweep). Output is projected to spark_conf/custom_tags so it is cloud-portable. +Cloud = true +CloudSlow = true + +RecordRequests = false + +Ignore = [ + "databricks.yml", + "edit.json", +] + +# The core count the backend puts in spark.master follows the node type. +[[Repls]] +Old = 'local\[\*, \d+\]' +New = 'local[*, [CORES]]' diff --git a/bundle/direct/dresources/configs/clusters.yml b/bundle/direct/dresources/configs/clusters.yml index b19d9c0483c..da471f866ee 100644 --- a/bundle/direct/dresources/configs/clusters.yml +++ b/bundle/direct/dresources/configs/clusters.yml @@ -51,6 +51,19 @@ backend_defaults: # See https://github.com/databricks/cli/issues/4418 - field: single_user_name + # is_single_node on a kind: CLASSIC_PREVIEW cluster makes the backend fill in the legacy + # single-node settings, which read back next to the flag. num_workers is forced to 0 as + # well, but that is the zero value and never diffs. Explicitly configured spark_conf and + # custom_tags keys are unaffected and still participate in drift detection. + # https://docs.databricks.com/api/workspace/clusters/create#is_single_node + # https://github.com/databricks/cli/issues/7563 + - field: spark_conf['spark.databricks.cluster.profile'] + values: ["singleNode"] + # The core count in local[*, N] follows the node type, so the value cannot be pinned. + - field: spark_conf['spark.master'] + - field: custom_tags['ResourceClass'] + values: ["SingleNode"] + # We have custom handler for this in cluster.go # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/clusters/resource_cluster.go#L109-L118 # DataSecurityModeDiffSuppressFunc: suppress when old != "" && new == "" diff --git a/libs/testserver/clusters.go b/libs/testserver/clusters.go index e0a046031e6..1451efb9317 100644 --- a/libs/testserver/clusters.go +++ b/libs/testserver/clusters.go @@ -179,6 +179,28 @@ func clusterFixUps(cluster *compute.ClusterDetails) { if cluster.DriverNodeTypeId == "" && cluster.NodeTypeId != "" { cluster.DriverNodeTypeId = cluster.NodeTypeId } + + applySingleNodeDefaults(cluster) +} + +// applySingleNodeDefaults mirrors the Clusters API expanding is_single_node on a CLASSIC_PREVIEW +// cluster into the legacy single-node settings, which then read back next to the flag. +// num_workers is also forced to 0 but that is the Go zero value already. The Jobs API does not +// do this for new_cluster specs: they read back as authored. +// https://docs.databricks.com/api/workspace/clusters/create#is_single_node +func applySingleNodeDefaults(cluster *compute.ClusterDetails) { + if !cluster.IsSingleNode || cluster.Kind != compute.KindClassicPreview { + return + } + if cluster.SparkConf == nil { + cluster.SparkConf = map[string]string{} + } + cluster.SparkConf["spark.databricks.cluster.profile"] = "singleNode" + cluster.SparkConf["spark.master"] = "local[*, 4]" + if cluster.CustomTags == nil { + cluster.CustomTags = map[string]string{} + } + cluster.CustomTags["ResourceClass"] = "SingleNode" } func (s *FakeWorkspace) ClustersGet(req Request, clusterId string) any {