Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7580.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Stop planning a spurious update for Lakeflow Connect pipelines whose `ingestion_definition.data_staging_options` is filled in by the backend. ([#7603](https://github.com/databricks/cli/pull/7603))
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
bundle:
name: pipeline-data-staging-options

resources:
pipelines:
implicit_staging:
name: cdc-implicit-staging
catalog: my_catalog
schema: my_schema
continuous: true
ingestion_definition:
connection_name: my_sqlserver_connection
connector_type: CDC
objects:
- table:
source_catalog: src_db
source_schema: dbo
source_table: my_table
destination_catalog: my_catalog
destination_schema: my_schema

explicit_staging:
name: cdc-explicit-staging
catalog: my_catalog
schema: my_schema
ingestion_definition:
connection_name: my_sqlserver_connection
connector_type: CDC
data_staging_options:
catalog_name: staging_catalog
schema_name: staging_schema
volume_name: staging_volume
objects:
- table:
source_catalog: src_db
source_schema: dbo
source_table: my_table
destination_catalog: my_catalog
destination_schema: my_schema
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@

=== Initial deployment

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-data-staging-options/default/files...
Created pipelines.explicit_staging
Created pipelines.implicit_staging
Files: 1 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

=== Plan is a no-op: the backend-filled data_staging_options is a default, not drift

>>> [CLI] bundle plan
Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged

>>> [CLI] bundle plan -o json
{
"resources.pipelines.explicit_staging": {
"action": "skip",
"changes": null
},
"resources.pipelines.implicit_staging": {
"action": "skip",
"changes": {
"ingestion_definition.data_staging_options": {
"action": "skip",
"reason": "backend_default",
"remote": {
"catalog_name": "my_catalog",
"schema_name": "my_schema",
"volume_name": "__databricks_ingestion_gateway_staging_data-[UUID]"
}
}
}
}
}

=== Redeploy is a no-op (no update call)

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-data-staging-options/default/files...
Files: 0 uploaded, 0 deleted
Resources: 0 created, 0 changed, 0 deleted, 2 unchanged

>>> print_requests.py --sort //api/2.0/pipelines
json[1].body.ingestion_definition.data_staging_options.catalog_name = "staging_catalog";
json[1].body.ingestion_definition.data_staging_options.schema_name = "staging_schema";
json[1].body.ingestion_definition.data_staging_options.volume_name = "staging_volume";

=== Explicitly configured data_staging_options still detects a remote change

>>> [CLI] bundle plan
update pipelines.explicit_staging

Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged

>>> [CLI] bundle plan -o json
{
"resources.pipelines.explicit_staging": {
"action": "update",
"changes": {
"ingestion_definition.data_staging_options.volume_name": {
"action": "update",
"old": "staging_volume",
"new": "staging_volume",
"remote": "changed_volume"
}
}
},
"resources.pipelines.implicit_staging": {
"action": "skip",
"changes": {
"ingestion_definition.data_staging_options": {
"action": "skip",
"reason": "backend_default",
"remote": {
"catalog_name": "my_catalog",
"schema_name": "my_schema",
"volume_name": "__databricks_ingestion_gateway_staging_data-[UUID]"
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@

title "Initial deployment"
trace $CLI bundle deploy

title "Plan is a no-op: the backend-filled data_staging_options is a default, not drift"
trace $CLI bundle plan
trace $CLI bundle plan -o json | nostamp | jq '.plan | map_values({action, changes})'

title "Redeploy is a no-op (no update call)"
trace $CLI bundle deploy
trace print_requests.py --sort //api/2.0/pipelines | contains.py '!"method": "PUT"' | gron.py | grep data_staging_options

title "Explicitly configured data_staging_options still detects a remote change"
explicit_id=$(read_id.py explicit_staging)
edit_resource.py pipelines $explicit_id <<EOF
r["ingestion_definition"]["data_staging_options"]["volume_name"] = "changed_volume"
EOF
trace $CLI bundle plan
trace $CLI bundle plan -o json | nostamp | jq '.plan | map_values({action, changes})'
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Local only: a Lakeflow Connect pipeline needs a real UC connection object, which the
# CI workspaces do not have. The fake server mirrors the backend by filling in
# data_staging_options on create/update when ingestion_definition is set.
RecordRequests = true
6 changes: 6 additions & 0 deletions bundle/direct/dresources/configs/pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,3 +70,9 @@ backend_defaults:
# s.SchemaPath("event_log", "schema").SetComputed()
- field: event_log.catalog
- field: event_log.schema

# Lakeflow Connect fills in the staging location when it is omitted: the destination
# catalog/schema and a __databricks_ingestion_gateway_staging_data-<pipeline-id> volume.
# The value embeds the pipeline's own id, so no config can match it and it cannot be
# pinned with values. An explicitly configured data_staging_options still compares as usual.
- field: ingestion_definition.data_staging_options
11 changes: 11 additions & 0 deletions libs/testserver/pipelines.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,17 @@ func setSpecDefaults(spec *pipelines.PipelineSpec, pipelineId string) {
if spec.Storage == "" && spec.Catalog == "" {
spec.Storage = "dbfs:/pipelines/" + pipelineId
}
// Lakeflow Connect fills in the staging location when it is omitted: the pipeline's
// destination catalog/schema plus a volume named after the pipeline id, so the config
// can never match it. See DataStagingOptions.VolumeName in the SDK. The volume is
// created in the destination catalog, so there is nothing to fill in without one.
if spec.IngestionDefinition != nil && spec.IngestionDefinition.DataStagingOptions == nil && spec.Catalog != "" {
spec.IngestionDefinition.DataStagingOptions = &pipelines.DataStagingOptions{
CatalogName: spec.Catalog,
SchemaName: spec.Schema,
VolumeName: "__databricks_ingestion_gateway_staging_data-" + pipelineId,
}
}
}

func (s *FakeWorkspace) PipelineUpdate(req Request, pipelineId string) Response {
Expand Down
Loading