diff --git a/.nextchanges/bundles/7580.md b/.nextchanges/bundles/7580.md new file mode 100644 index 00000000000..fe135f5550d --- /dev/null +++ b/.nextchanges/bundles/7580.md @@ -0,0 +1 @@ +* Stop planning a spurious update for Lakeflow Connect pipelines whose `ingestion_definition.data_staging_options` is filled in by the backend. ([#7603](https://github.com/databricks/cli/pull/7603)) diff --git a/acceptance/bundle/resources/pipelines/drift/data_staging_options/databricks.yml b/acceptance/bundle/resources/pipelines/drift/data_staging_options/databricks.yml new file mode 100644 index 00000000000..ff87052406d --- /dev/null +++ b/acceptance/bundle/resources/pipelines/drift/data_staging_options/databricks.yml @@ -0,0 +1,39 @@ +bundle: + name: pipeline-data-staging-options + +resources: + pipelines: + implicit_staging: + name: cdc-implicit-staging + catalog: my_catalog + schema: my_schema + continuous: true + ingestion_definition: + connection_name: my_sqlserver_connection + connector_type: CDC + objects: + - table: + source_catalog: src_db + source_schema: dbo + source_table: my_table + destination_catalog: my_catalog + destination_schema: my_schema + + explicit_staging: + name: cdc-explicit-staging + catalog: my_catalog + schema: my_schema + ingestion_definition: + connection_name: my_sqlserver_connection + connector_type: CDC + data_staging_options: + catalog_name: staging_catalog + schema_name: staging_schema + volume_name: staging_volume + objects: + - table: + source_catalog: src_db + source_schema: dbo + source_table: my_table + destination_catalog: my_catalog + destination_schema: my_schema diff --git a/acceptance/bundle/resources/pipelines/drift/data_staging_options/out.test.toml b/acceptance/bundle/resources/pipelines/drift/data_staging_options/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/drift/data_staging_options/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/drift/data_staging_options/output.txt b/acceptance/bundle/resources/pipelines/drift/data_staging_options/output.txt new file mode 100644 index 00000000000..c7064279a54 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/drift/data_staging_options/output.txt @@ -0,0 +1,84 @@ + +=== Initial deployment + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-data-staging-options/default/files... +Created pipelines.explicit_staging +Created pipelines.implicit_staging +Files: 1 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Plan is a no-op: the backend-filled data_staging_options is a default, not drift + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle plan -o json +{ + "resources.pipelines.explicit_staging": { + "action": "skip", + "changes": null + }, + "resources.pipelines.implicit_staging": { + "action": "skip", + "changes": { + "ingestion_definition.data_staging_options": { + "action": "skip", + "reason": "backend_default", + "remote": { + "catalog_name": "my_catalog", + "schema_name": "my_schema", + "volume_name": "__databricks_ingestion_gateway_staging_data-[UUID]" + } + } + } + } +} + +=== Redeploy is a no-op (no update call) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-data-staging-options/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py --sort //api/2.0/pipelines +json[1].body.ingestion_definition.data_staging_options.catalog_name = "staging_catalog"; +json[1].body.ingestion_definition.data_staging_options.schema_name = "staging_schema"; +json[1].body.ingestion_definition.data_staging_options.volume_name = "staging_volume"; + +=== Explicitly configured data_staging_options still detects a remote change + +>>> [CLI] bundle plan +update pipelines.explicit_staging + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle plan -o json +{ + "resources.pipelines.explicit_staging": { + "action": "update", + "changes": { + "ingestion_definition.data_staging_options.volume_name": { + "action": "update", + "old": "staging_volume", + "new": "staging_volume", + "remote": "changed_volume" + } + } + }, + "resources.pipelines.implicit_staging": { + "action": "skip", + "changes": { + "ingestion_definition.data_staging_options": { + "action": "skip", + "reason": "backend_default", + "remote": { + "catalog_name": "my_catalog", + "schema_name": "my_schema", + "volume_name": "__databricks_ingestion_gateway_staging_data-[UUID]" + } + } + } + } +} diff --git a/acceptance/bundle/resources/pipelines/drift/data_staging_options/script b/acceptance/bundle/resources/pipelines/drift/data_staging_options/script new file mode 100644 index 00000000000..67cace20e9d --- /dev/null +++ b/acceptance/bundle/resources/pipelines/drift/data_staging_options/script @@ -0,0 +1,19 @@ + +title "Initial deployment" +trace $CLI bundle deploy + +title "Plan is a no-op: the backend-filled data_staging_options is a default, not drift" +trace $CLI bundle plan +trace $CLI bundle plan -o json | nostamp | jq '.plan | map_values({action, changes})' + +title "Redeploy is a no-op (no update call)" +trace $CLI bundle deploy +trace print_requests.py --sort //api/2.0/pipelines | contains.py '!"method": "PUT"' | gron.py | grep data_staging_options + +title "Explicitly configured data_staging_options still detects a remote change" +explicit_id=$(read_id.py explicit_staging) +edit_resource.py pipelines $explicit_id < volume. + # The value embeds the pipeline's own id, so no config can match it and it cannot be + # pinned with values. An explicitly configured data_staging_options still compares as usual. + - field: ingestion_definition.data_staging_options diff --git a/libs/testserver/pipelines.go b/libs/testserver/pipelines.go index 6960aebf350..684112d7de0 100644 --- a/libs/testserver/pipelines.go +++ b/libs/testserver/pipelines.go @@ -96,6 +96,17 @@ func setSpecDefaults(spec *pipelines.PipelineSpec, pipelineId string) { if spec.Storage == "" && spec.Catalog == "" { spec.Storage = "dbfs:/pipelines/" + pipelineId } + // Lakeflow Connect fills in the staging location when it is omitted: the pipeline's + // destination catalog/schema plus a volume named after the pipeline id, so the config + // can never match it. See DataStagingOptions.VolumeName in the SDK. The volume is + // created in the destination catalog, so there is nothing to fill in without one. + if spec.IngestionDefinition != nil && spec.IngestionDefinition.DataStagingOptions == nil && spec.Catalog != "" { + spec.IngestionDefinition.DataStagingOptions = &pipelines.DataStagingOptions{ + CatalogName: spec.Catalog, + SchemaName: spec.Schema, + VolumeName: "__databricks_ingestion_gateway_staging_data-" + pipelineId, + } + } } func (s *FakeWorkspace) PipelineUpdate(req Request, pipelineId string) Response {