From f6772b79ddfc30cdd259541036a988dfae71dc7a Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Fri, 13 Nov 2026 14:29:16 +0000 Subject: [PATCH 1/2] direct: treat pipeline data_staging_options as a backend default A Lakeflow Connect pipeline that omits ingestion_definition.data_staging_options was planned as "update" on every `bundle plan` and updated on every deploy. The backend fills the field with the destination catalog/schema and a volume named __databricks_ingestion_gateway_staging_data-. The direct engine saw a remote-only value with no matching rule and reported it as drift, although no config can ever match it because it embeds the pipeline's own id. Continuous pipelines were restarted by every deploy because of this. Add a `backend_defaults` rule without pinned values for ingestion_definition.data_staging_options in pipelines.yml. It only fires when the field is absent from both state and config, so an explicitly configured data_staging_options still participates in drift detection. Teach the fake server to fill in data_staging_options like the backend when ingestion_definition is set, the field is omitted, and a destination catalog exists, and add an acceptance test covering both the implicit (no-op plan) and explicit (remote change reported) cases. Fixes #7580 Co-authored-by: Isaac --- .nextchanges/bundles/7580.md | 1 + .../drift/data_staging_options/databricks.yml | 39 +++++++++ .../drift/data_staging_options/out.test.toml | 2 + .../drift/data_staging_options/output.txt | 84 +++++++++++++++++++ .../drift/data_staging_options/script | 19 +++++ .../drift/data_staging_options/test.toml | 4 + .../direct/dresources/configs/pipelines.yml | 6 ++ libs/testserver/pipelines.go | 11 +++ 8 files changed, 166 insertions(+) create mode 100644 .nextchanges/bundles/7580.md create mode 100644 acceptance/bundle/resources/pipelines/drift/data_staging_options/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/drift/data_staging_options/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/drift/data_staging_options/output.txt create mode 100644 acceptance/bundle/resources/pipelines/drift/data_staging_options/script create mode 100644 acceptance/bundle/resources/pipelines/drift/data_staging_options/test.toml diff --git a/.nextchanges/bundles/7580.md b/.nextchanges/bundles/7580.md new file mode 100644 index 00000000000..102e6819a6d --- /dev/null +++ b/.nextchanges/bundles/7580.md @@ -0,0 +1 @@ +* Stop planning a spurious update for Lakeflow Connect pipelines whose `ingestion_definition.data_staging_options` is filled in by the backend. ([#NNNN](https://github.com/databricks/cli/pull/NNNN)) diff --git a/acceptance/bundle/resources/pipelines/drift/data_staging_options/databricks.yml b/acceptance/bundle/resources/pipelines/drift/data_staging_options/databricks.yml new file mode 100644 index 00000000000..ff87052406d --- /dev/null +++ b/acceptance/bundle/resources/pipelines/drift/data_staging_options/databricks.yml @@ -0,0 +1,39 @@ +bundle: + name: pipeline-data-staging-options + +resources: + pipelines: + implicit_staging: + name: cdc-implicit-staging + catalog: my_catalog + schema: my_schema + continuous: true + ingestion_definition: + connection_name: my_sqlserver_connection + connector_type: CDC + objects: + - table: + source_catalog: src_db + source_schema: dbo + source_table: my_table + destination_catalog: my_catalog + destination_schema: my_schema + + explicit_staging: + name: cdc-explicit-staging + catalog: my_catalog + schema: my_schema + ingestion_definition: + connection_name: my_sqlserver_connection + connector_type: CDC + data_staging_options: + catalog_name: staging_catalog + schema_name: staging_schema + volume_name: staging_volume + objects: + - table: + source_catalog: src_db + source_schema: dbo + source_table: my_table + destination_catalog: my_catalog + destination_schema: my_schema diff --git a/acceptance/bundle/resources/pipelines/drift/data_staging_options/out.test.toml b/acceptance/bundle/resources/pipelines/drift/data_staging_options/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/drift/data_staging_options/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/drift/data_staging_options/output.txt b/acceptance/bundle/resources/pipelines/drift/data_staging_options/output.txt new file mode 100644 index 00000000000..c7064279a54 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/drift/data_staging_options/output.txt @@ -0,0 +1,84 @@ + +=== Initial deployment + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-data-staging-options/default/files... +Created pipelines.explicit_staging +Created pipelines.implicit_staging +Files: 1 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Plan is a no-op: the backend-filled data_staging_options is a default, not drift + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle plan -o json +{ + "resources.pipelines.explicit_staging": { + "action": "skip", + "changes": null + }, + "resources.pipelines.implicit_staging": { + "action": "skip", + "changes": { + "ingestion_definition.data_staging_options": { + "action": "skip", + "reason": "backend_default", + "remote": { + "catalog_name": "my_catalog", + "schema_name": "my_schema", + "volume_name": "__databricks_ingestion_gateway_staging_data-[UUID]" + } + } + } + } +} + +=== Redeploy is a no-op (no update call) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-data-staging-options/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py --sort //api/2.0/pipelines +json[1].body.ingestion_definition.data_staging_options.catalog_name = "staging_catalog"; +json[1].body.ingestion_definition.data_staging_options.schema_name = "staging_schema"; +json[1].body.ingestion_definition.data_staging_options.volume_name = "staging_volume"; + +=== Explicitly configured data_staging_options still detects a remote change + +>>> [CLI] bundle plan +update pipelines.explicit_staging + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle plan -o json +{ + "resources.pipelines.explicit_staging": { + "action": "update", + "changes": { + "ingestion_definition.data_staging_options.volume_name": { + "action": "update", + "old": "staging_volume", + "new": "staging_volume", + "remote": "changed_volume" + } + } + }, + "resources.pipelines.implicit_staging": { + "action": "skip", + "changes": { + "ingestion_definition.data_staging_options": { + "action": "skip", + "reason": "backend_default", + "remote": { + "catalog_name": "my_catalog", + "schema_name": "my_schema", + "volume_name": "__databricks_ingestion_gateway_staging_data-[UUID]" + } + } + } + } +} diff --git a/acceptance/bundle/resources/pipelines/drift/data_staging_options/script b/acceptance/bundle/resources/pipelines/drift/data_staging_options/script new file mode 100644 index 00000000000..67cace20e9d --- /dev/null +++ b/acceptance/bundle/resources/pipelines/drift/data_staging_options/script @@ -0,0 +1,19 @@ + +title "Initial deployment" +trace $CLI bundle deploy + +title "Plan is a no-op: the backend-filled data_staging_options is a default, not drift" +trace $CLI bundle plan +trace $CLI bundle plan -o json | nostamp | jq '.plan | map_values({action, changes})' + +title "Redeploy is a no-op (no update call)" +trace $CLI bundle deploy +trace print_requests.py --sort //api/2.0/pipelines | contains.py '!"method": "PUT"' | gron.py | grep data_staging_options + +title "Explicitly configured data_staging_options still detects a remote change" +explicit_id=$(read_id.py explicit_staging) +edit_resource.py pipelines $explicit_id < volume. + # The value embeds the pipeline's own id, so no config can match it and it cannot be + # pinned with values. An explicitly configured data_staging_options still compares as usual. + - field: ingestion_definition.data_staging_options diff --git a/libs/testserver/pipelines.go b/libs/testserver/pipelines.go index 6960aebf350..684112d7de0 100644 --- a/libs/testserver/pipelines.go +++ b/libs/testserver/pipelines.go @@ -96,6 +96,17 @@ func setSpecDefaults(spec *pipelines.PipelineSpec, pipelineId string) { if spec.Storage == "" && spec.Catalog == "" { spec.Storage = "dbfs:/pipelines/" + pipelineId } + // Lakeflow Connect fills in the staging location when it is omitted: the pipeline's + // destination catalog/schema plus a volume named after the pipeline id, so the config + // can never match it. See DataStagingOptions.VolumeName in the SDK. The volume is + // created in the destination catalog, so there is nothing to fill in without one. + if spec.IngestionDefinition != nil && spec.IngestionDefinition.DataStagingOptions == nil && spec.Catalog != "" { + spec.IngestionDefinition.DataStagingOptions = &pipelines.DataStagingOptions{ + CatalogName: spec.Catalog, + SchemaName: spec.Schema, + VolumeName: "__databricks_ingestion_gateway_staging_data-" + pipelineId, + } + } } func (s *FakeWorkspace) PipelineUpdate(req Request, pipelineId string) Response { From 9d316654c86d4d732d511f0cd48b95ad9f423384 Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Fri, 13 Nov 2026 14:36:51 +0000 Subject: [PATCH 2/2] Link changelog fragment to PR #7603 --- .nextchanges/bundles/7580.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/7580.md b/.nextchanges/bundles/7580.md index 102e6819a6d..fe135f5550d 100644 --- a/.nextchanges/bundles/7580.md +++ b/.nextchanges/bundles/7580.md @@ -1 +1 @@ -* Stop planning a spurious update for Lakeflow Connect pipelines whose `ingestion_definition.data_staging_options` is filled in by the backend. ([#NNNN](https://github.com/databricks/cli/pull/NNNN)) +* Stop planning a spurious update for Lakeflow Connect pipelines whose `ingestion_definition.data_staging_options` is filled in by the backend. ([#7603](https://github.com/databricks/cli/pull/7603))