Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7563.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Fix `bundle plan` reporting the `spark_conf` and `custom_tags` the backend derives from `is_single_node: true` as drift on clusters. ([#7577](https://github.com/databricks/cli/pull/7577))
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
bundle:
name: test-bundle-$UNIQUE_NAME

workspace:
root_path: ~/.bundle/$UNIQUE_NAME

resources:
clusters:
# No spark_conf at all: the backend adds the whole map from is_single_node.
implicit:
cluster_name: test-cluster-implicit-$UNIQUE_NAME
spark_version: $DEFAULT_SPARK_VERSION
node_type_id: $NODE_TYPE_ID
is_single_node: true
kind: CLASSIC_PREVIEW
data_security_mode: SINGLE_USER
autotermination_minutes: 10

# An explicit spark_conf key next to the backend-added ones: it must still be
# compared against the remote.
explicit:
cluster_name: test-cluster-explicit-$UNIQUE_NAME
spark_version: $DEFAULT_SPARK_VERSION
node_type_id: $NODE_TYPE_ID
is_single_node: true
kind: CLASSIC_PREVIEW
data_security_mode: SINGLE_USER
autotermination_minutes: 10
spark_conf:
spark.sql.shuffle.partitions: "8"
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Cloud = true
CloudSlow = true
EnvMatrix.DMS = ["", "true"]
90 changes: 90 additions & 0 deletions acceptance/bundle/resources/clusters/single_node/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
Created clusters.explicit
Created clusters.implicit
Files: 2 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

=== Settings the backend derives from is_single_node are not drift

>>> [CLI] bundle plan
Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged

=== ...classified backend_default

>>> [CLI] bundle plan -o json
{
"resources.clusters.explicit": {
"custom_tags": {
"action": "skip",
"reason": "backend_default",
"remote": {
"ResourceClass": "SingleNode"
}
},
"spark_conf['spark.databricks.cluster.profile']": {
"action": "skip",
"reason": "backend_default",
"remote": "singleNode"
},
"spark_conf['spark.master']": {
"action": "skip",
"reason": "backend_default",
"remote": "local[*, [CORES]]"
}
},
"resources.clusters.implicit": {
"custom_tags": {
"action": "skip",
"reason": "backend_default",
"remote": {
"ResourceClass": "SingleNode"
}
},
"spark_conf": {
"action": "skip",
"reason": "backend_default",
"remote": {
"spark.databricks.cluster.profile": "singleNode",
"spark.master": "local[*, [CORES]]"
}
}
}
}

=== An explicitly configured spark_conf key changed out-of-band is still drift

>>> [CLI] bundle plan
update clusters.explicit

Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged

>>> [CLI] bundle plan -o json
{
"spark_conf['spark.databricks.cluster.profile']": {
"action": "skip",
"reason": "backend_default",
"remote": "singleNode"
},
"spark_conf['spark.master']": {
"action": "skip",
"reason": "backend_default",
"remote": "local[*, [CORES]]"
},
"spark_conf['spark.sql.shuffle.partitions']": {
"action": "update",
"new": "8",
"old": "8",
"remote": "16"
}
}

>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
delete resources.clusters.explicit
delete resources.clusters.implicit

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]

Destroy: 2 deleted
27 changes: 27 additions & 0 deletions acceptance/bundle/resources/clusters/single_node/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
envsubst < databricks.yml.tmpl > databricks.yml

cleanup() {
trace $CLI bundle destroy --auto-approve
}
trap cleanup EXIT

trace $CLI bundle deploy

title "Settings the backend derives from is_single_node are not drift"
trace $CLI bundle plan

title "...classified backend_default"
trace $CLI bundle plan -o json | jq -S '.plan | with_entries(.value |= (.changes // {}
| with_entries(select(.key | test("spark_conf|custom_tags")))))'

title "An explicitly configured spark_conf key changed out-of-band is still drift"
explicit_id=$(read_id.py explicit)
# Re-send only the authored spark_conf keys: the backend-derived ones come back from is_single_node.
$CLI clusters get "$explicit_id" | jq '{cluster_id, cluster_name, spark_version, node_type_id,
is_single_node, kind, data_security_mode, autotermination_minutes,
spark_conf: (.spark_conf | del(.["spark.databricks.cluster.profile"], .["spark.master"])
+ {"spark.sql.shuffle.partitions": "16"})}' > edit.json
trace $CLI clusters edit --json @edit.json &> LOG.edit
trace $CLI bundle plan
trace $CLI bundle plan -o json | jq -S '.plan["resources.clusters.explicit"].changes
| with_entries(select(.key | test("spark_conf")))'
17 changes: 17 additions & 0 deletions acceptance/bundle/resources/clusters/single_node/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# The standalone clusters resource boots a real cluster at deploy, which is slow and has hit
# capacity limits on cloud; CloudSlow keeps it out of the per-PR run (it still runs locally and
# in the full cloud sweep). Output is projected to spark_conf/custom_tags so it is cloud-portable.
Cloud = true
CloudSlow = true

RecordRequests = false

Ignore = [
"databricks.yml",
"edit.json",
]

# The core count the backend puts in spark.master follows the node type.
[[Repls]]
Old = 'local\[\*, \d+\]'
New = 'local[*, [CORES]]'
13 changes: 13 additions & 0 deletions bundle/direct/dresources/configs/clusters.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,19 @@ backend_defaults:
# See https://github.com/databricks/cli/issues/4418
- field: single_user_name

# is_single_node on a kind: CLASSIC_PREVIEW cluster makes the backend fill in the legacy
# single-node settings, which read back next to the flag. num_workers is forced to 0 as
# well, but that is the zero value and never diffs. Explicitly configured spark_conf and
# custom_tags keys are unaffected and still participate in drift detection.
# https://docs.databricks.com/api/workspace/clusters/create#is_single_node
# https://github.com/databricks/cli/issues/7563
- field: spark_conf['spark.databricks.cluster.profile']
values: ["singleNode"]
# The core count in local[*, N] follows the node type, so the value cannot be pinned.
- field: spark_conf['spark.master']
- field: custom_tags['ResourceClass']
values: ["SingleNode"]

# We have custom handler for this in cluster.go
# https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/clusters/resource_cluster.go#L109-L118
# DataSecurityModeDiffSuppressFunc: suppress when old != "" && new == ""
Expand Down
22 changes: 22 additions & 0 deletions libs/testserver/clusters.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,28 @@ func clusterFixUps(cluster *compute.ClusterDetails) {
if cluster.DriverNodeTypeId == "" && cluster.NodeTypeId != "" {
cluster.DriverNodeTypeId = cluster.NodeTypeId
}

applySingleNodeDefaults(cluster)
}

// applySingleNodeDefaults mirrors the Clusters API expanding is_single_node on a CLASSIC_PREVIEW
// cluster into the legacy single-node settings, which then read back next to the flag.
// num_workers is also forced to 0 but that is the Go zero value already. The Jobs API does not
// do this for new_cluster specs: they read back as authored.
// https://docs.databricks.com/api/workspace/clusters/create#is_single_node
func applySingleNodeDefaults(cluster *compute.ClusterDetails) {
if !cluster.IsSingleNode || cluster.Kind != compute.KindClassicPreview {
return
}
if cluster.SparkConf == nil {
cluster.SparkConf = map[string]string{}
}
cluster.SparkConf["spark.databricks.cluster.profile"] = "singleNode"
cluster.SparkConf["spark.master"] = "local[*, 4]"
if cluster.CustomTags == nil {
cluster.CustomTags = map[string]string{}
}
cluster.CustomTags["ResourceClass"] = "SingleNode"
}

func (s *FakeWorkspace) ClustersGet(req Request, clusterId string) any {
Expand Down
Loading