From 72a3c78b882a4f87a98496ede716a4eae5e10c2d Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 20 Oct 2026 20:50:44 +0200 Subject: [PATCH 1/9] Wait for grants to provision as part of depends on --- .../bundles/grants-before-dependents.md | 1 + .../pipeline_run_as/databricks.yml.tmpl | 34 +++++++ .../schemas/pipeline_run_as/out.test.toml | 2 + .../grants/schemas/pipeline_run_as/output.txt | 78 +++++++++++++++ .../grants/schemas/pipeline_run_as/script | 18 ++++ .../grants/schemas/pipeline_run_as/test.toml | 3 + .../schemas/pipeline_run_as/transformation.py | 1 + bundle/direct/bundle_plan.go | 63 +++++++++++++ libs/testserver/pipelines.go | 43 +++++++++ libs/testserver/pipelines_test.go | 94 +++++++++++++++++++ 10 files changed, 337 insertions(+) create mode 100644 .nextchanges/bundles/grants-before-dependents.md create mode 100644 acceptance/bundle/resources/grants/schemas/pipeline_run_as/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/grants/schemas/pipeline_run_as/out.test.toml create mode 100644 acceptance/bundle/resources/grants/schemas/pipeline_run_as/output.txt create mode 100644 acceptance/bundle/resources/grants/schemas/pipeline_run_as/script create mode 100644 acceptance/bundle/resources/grants/schemas/pipeline_run_as/test.toml create mode 100644 acceptance/bundle/resources/grants/schemas/pipeline_run_as/transformation.py diff --git a/.nextchanges/bundles/grants-before-dependents.md b/.nextchanges/bundles/grants-before-dependents.md new file mode 100644 index 00000000000..5070ca2d692 --- /dev/null +++ b/.nextchanges/bundles/grants-before-dependents.md @@ -0,0 +1 @@ +* Resources that reference another resource are now deployed after that resource's `grants`, fixing intermittent `User does not have CREATE TABLE on Schema` errors when a pipeline's `run_as` principal is granted access to its schema in the same bundle. diff --git a/acceptance/bundle/resources/grants/schemas/pipeline_run_as/databricks.yml.tmpl b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/databricks.yml.tmpl new file mode 100644 index 00000000000..f029e2d98a1 --- /dev/null +++ b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/databricks.yml.tmpl @@ -0,0 +1,34 @@ +bundle: + name: schema-grants-pipeline-$UNIQUE_NAME + +resources: + catalogs: + pipeline_catalog: + name: catalog_grants_pipeline_$UNIQUE_NAME + grants: + - principal: $RUN_AS_SP + privileges: + - USE_CATALOG + + schemas: + src: + catalog_name: ${resources.catalogs.pipeline_catalog.name} + name: src + grants: + - principal: $RUN_AS_SP + privileges: + - USE_SCHEMA + - CREATE_TABLE + - CREATE_MATERIALIZED_VIEW + + pipelines: + load: + name: schema-grants-pipeline-$UNIQUE_NAME + catalog: ${resources.schemas.src.catalog_name} + schema: ${resources.schemas.src.name} + serverless: true + libraries: + - file: + path: ./transformation.py + run_as: + service_principal_name: $RUN_AS_SP diff --git a/acceptance/bundle/resources/grants/schemas/pipeline_run_as/out.test.toml b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/grants/schemas/pipeline_run_as/output.txt b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/output.txt new file mode 100644 index 00000000000..9ec73822a86 --- /dev/null +++ b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/output.txt @@ -0,0 +1,78 @@ + +>>> [CLI] bundle plan -o json +{ + "resources.catalogs.pipeline_catalog": null, + "resources.catalogs.pipeline_catalog.grants": [ + { + "node": "resources.catalogs.pipeline_catalog", + "label": "${resources.catalogs.pipeline_catalog.id}" + } + ], + "resources.pipelines.load": [ + { + "node": "resources.schemas.src", + "label": "${resources.schemas.src.catalog_name}" + }, + { + "node": "resources.schemas.src", + "label": "${resources.schemas.src.name}" + }, + { + "node": "resources.schemas.src.grants", + "label": "${resources.schemas.src.catalog_name}" + } + ], + "resources.schemas.src": [ + { + "node": "resources.catalogs.pipeline_catalog", + "label": "${resources.catalogs.pipeline_catalog.name}" + }, + { + "node": "resources.catalogs.pipeline_catalog.grants", + "label": "${resources.catalogs.pipeline_catalog.name}" + } + ], + "resources.schemas.src.grants": [ + { + "node": "resources.schemas.src", + "label": "${resources.schemas.src.id}" + } + ] +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/schema-grants-pipeline-[UNIQUE_NAME]/default/files... +Created catalogs.pipeline_catalog +Created catalogs.pipeline_catalog.grants +Created pipelines.load +Created schemas.src +Created schemas.src.grants +Files: 3 uploaded, 0 deleted +Resources: 5 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --summary //unity-catalog/catalogs //unity-catalog/schemas //unity-catalog/permissions //pipelines +POST /api/2.1/unity-catalog/catalogs +PATCH /api/2.1/unity-catalog/permissions/catalog/catalog_grants_pipeline_[UNIQUE_NAME] +POST /api/2.1/unity-catalog/schemas +PATCH /api/2.1/unity-catalog/permissions/schema/catalog_grants_pipeline_[UNIQUE_NAME].src +POST /api/2.0/pipelines + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.pipeline_catalog + delete resources.pipelines.load + delete resources.schemas.src + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.src + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.load + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/schema-grants-pipeline-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/grants/schemas/pipeline_run_as/script b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/script new file mode 100644 index 00000000000..9ce8ea179b3 --- /dev/null +++ b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/script @@ -0,0 +1,18 @@ +# Creating the pipeline fails unless its run_as principal can already create tables in the +# schema, so the pipeline must be deployed after the grants it relies on. +# See https://github.com/databricks/cli/issues/4573 +export RUN_AS_SP="${TEST_SP_APPLICATION_ID:-6f1c2b3a-4d5e-4f60-8a7b-9c0d1e2f3a4b}" + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle plan -o json | jq '.plan | map_values(.depends_on)' + +trace $CLI bundle deploy +trace print_requests.py --summary //unity-catalog/catalogs //unity-catalog/schemas //unity-catalog/permissions //pipelines + +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/grants/schemas/pipeline_run_as/test.toml b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/test.toml new file mode 100644 index 00000000000..10a7e8b41bb --- /dev/null +++ b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/test.toml @@ -0,0 +1,3 @@ +# Local only: on cloud the deploying identity lacks the servicePrincipal.user role on +# TEST_SP_APPLICATION_ID, so it cannot set that service principal as the pipeline's run_as. +Cloud = false diff --git a/acceptance/bundle/resources/grants/schemas/pipeline_run_as/transformation.py b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/transformation.py new file mode 100644 index 00000000000..7df869a15e7 --- /dev/null +++ b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/transformation.py @@ -0,0 +1 @@ +print("Hello, World!") diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 43a58291ab2..f2de4ad918e 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -1771,6 +1771,8 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root p.Plan[node] = &e } + addGrantsDependencies(p) + // lifecycle.depends_on only orders resources: a target that is not in the plan // (e.g. skipped as empty) needs no ordering. for _, entry := range p.Plan { @@ -1793,6 +1795,67 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root return p, nil } +func compareDependsOn(a, b deployplan.DependsOnEntry) int { + if a.Node != b.Node { + return strings.Compare(a.Node, b.Node) + } + return strings.Compare(a.Label, b.Label) +} + +// addGrantsDependencies makes resources that refer to another resource also depend on that +// resource's grants, because creating some resources requires the grants to be in place: a +// pipeline cannot be created before its run_as principal has CREATE TABLE on the target schema. +// A grants dependency that would create a cycle is skipped, e.g. when a schema grants privileges +// to the service principal of an app that refers to the schema. +func addGrantsDependencies(plan *deployplan.Plan) { + for _, node := range slices.Sorted(maps.Keys(plan.Plan)) { + entry := plan.Plan[node] + + seen := make(map[string]bool, len(entry.DependsOn)) + for _, dep := range entry.DependsOn { + seen[dep.Node] = true + } + + var grantsDeps []deployplan.DependsOnEntry + for _, dep := range entry.DependsOn { + grantsNode := dep.Node + ".grants" + if seen[grantsNode] || plan.Plan[grantsNode] == nil || dependsOnTransitively(plan, grantsNode, node) { + continue + } + seen[grantsNode] = true + grantsDeps = append(grantsDeps, deployplan.DependsOnEntry{Node: grantsNode, Label: dep.Label}) + } + + if len(grantsDeps) > 0 { + entry.DependsOn = append(entry.DependsOn, grantsDeps...) + slices.SortFunc(entry.DependsOn, compareDependsOn) + } + } +} + +// dependsOnTransitively reports whether node is target or depends on it, directly or through other nodes. +func dependsOnTransitively(plan *deployplan.Plan, node, target string) bool { + visited := make(map[string]bool) + stack := []string{node} + for len(stack) > 0 { + n := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if n == target { + return true + } + // A dependency on a node missing from the plan is reported by makeGraph. + entry := plan.Plan[n] + if visited[n] || entry == nil { + continue + } + visited[n] = true + for _, dep := range entry.DependsOn { + stack = append(stack, dep.Node) + } + } + return false +} + // ExtractReferences extracts variable references from the config subtree rooted at node, // keeping only those whose field path exists in stateType (references in input-only or // bundle:"readonly" fields, such as volumes' computed volume_path, are skipped). diff --git a/libs/testserver/pipelines.go b/libs/testserver/pipelines.go index 6960aebf350..37b7668ad59 100644 --- a/libs/testserver/pipelines.go +++ b/libs/testserver/pipelines.go @@ -1,10 +1,13 @@ package testserver import ( + "cmp" "encoding/json" "fmt" + "slices" "strings" + "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/pipelines" ) @@ -64,6 +67,18 @@ func (s *FakeWorkspace) PipelineCreate(req Request) Response { } r.Parameters = create.Parameters + // "schema" and "target" both name the target schema; only one of them can be set. + schemaName := spec.Catalog + "." + cmp.Or(spec.Schema, spec.Target) + if !s.runAsCanCreateTables(schemaName, create.RunAs) { + return Response{ + StatusCode: 403, + Body: map[string]string{ + "error_code": permissionDeniedErrorCode, + "message": fmt.Sprintf("User does not have CREATE TABLE on Schema '%s'.", schemaName), + }, + } + } + pipelineId := nextUUID() r.PipelineId = pipelineId r.CreatorUserName = "tester@databricks.com" @@ -88,6 +103,34 @@ func (s *FakeWorkspace) PipelineCreate(req Request) Response { } } +// runAsCanCreateTables reports whether the run_as principal of a new pipeline can create +// tables in its target schema, which the backend requires to create the pipeline. Only +// schemas created in this workspace are checked, because grants are tracked only for them. +func (s *FakeWorkspace) runAsCanCreateTables(schemaName string, runAs *pipelines.RunAs) bool { + var principal string + if runAs != nil { + principal = cmp.Or(runAs.ServicePrincipalName, runAs.UserName) + } + + schema, ok := s.Schemas[schemaName] + if principal == "" || !ok || principal == schema.Owner { + return true + } + + // Privileges granted on the catalog are inherited by its schemas. + for _, key := range []string{grantsKey("catalog", schema.CatalogName), grantsKey("schema", schemaName)} { + for _, assignment := range s.Grants[key] { + if assignment.Principal != principal { + continue + } + if slices.Contains(assignment.Privileges, catalog.PrivilegeCreateTable) || slices.Contains(assignment.Privileges, catalog.PrivilegeAllPrivileges) { + return true + } + } + } + return false +} + func setSpecDefaults(spec *pipelines.PipelineSpec, pipelineId string) { spec.Id = pipelineId // If the pipeline definition does not specify a catalog, it switches to Hive metastore mode diff --git a/libs/testserver/pipelines_test.go b/libs/testserver/pipelines_test.go index 6bb1dadfbdb..12f0b2a07c6 100644 --- a/libs/testserver/pipelines_test.go +++ b/libs/testserver/pipelines_test.go @@ -1,6 +1,7 @@ package testserver import ( + "strings" "testing" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -48,6 +49,99 @@ func TestPipelineCreate_AllowsSingleSegmentTargetSchemaName(t *testing.T) { assert.Equal(t, 0, response.StatusCode) } +func TestPipelineCreate_RequiresRunAsCreateTableOnSchema(t *testing.T) { + const pipeline = `{"name": "p", "catalog": "main", "schema": "src", "run_as": {"service_principal_name": "sp"}}` + + tests := []struct { + name string + createSchema bool + grants map[string]string + pipeline string + expectedStatusCode int + }{ + { + name: "no grants", + createSchema: true, + pipeline: pipeline, + expectedStatusCode: 403, + }, + { + name: "no grants with target", + createSchema: true, + pipeline: `{"name": "p", "catalog": "main", "target": "src", "run_as": {"service_principal_name": "sp"}}`, + expectedStatusCode: 403, + }, + { + name: "grants without CREATE_TABLE", + createSchema: true, + grants: map[string]string{"schema/main.src": "USE_SCHEMA"}, + pipeline: pipeline, + expectedStatusCode: 403, + }, + { + name: "CREATE_TABLE on schema", + createSchema: true, + grants: map[string]string{"schema/main.src": "CREATE_TABLE"}, + pipeline: pipeline, + }, + { + name: "ALL_PRIVILEGES on schema", + createSchema: true, + grants: map[string]string{"schema/main.src": "ALL_PRIVILEGES"}, + pipeline: pipeline, + }, + { + name: "CREATE_TABLE on catalog", + createSchema: true, + grants: map[string]string{"catalog/main": "CREATE_TABLE"}, + pipeline: pipeline, + }, + { + name: "run_as schema owner", + createSchema: true, + pipeline: `{"name": "p", "catalog": "main", "schema": "src", "run_as": {"user_name": "` + TestUser.UserName + `"}}`, + }, + { + name: "no run_as", + createSchema: true, + pipeline: `{"name": "p", "catalog": "main", "schema": "src"}`, + }, + { + name: "schema not created in workspace", + pipeline: pipeline, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // A user token, so that the schema owner is TestUser. + workspace := NewFakeWorkspace("http://test", UserNameTokenPrefix+"123") + + if tt.createSchema { + response := workspace.SchemasCreate(Request{Body: []byte(`{"catalog_name": "main", "name": "src"}`)}) + require.Equal(t, 0, response.StatusCode) + } + + for securable, privilege := range tt.grants { + securableType, fullName, _ := strings.Cut(securable, "/") + response := workspace.GrantsUpdate(Request{ + Body: []byte(`{"changes": [{"principal": "sp", "add": ["` + privilege + `"]}]}`), + }, securableType, fullName) + require.Equal(t, 0, response.StatusCode) + } + + response := workspace.PipelineCreate(Request{Body: []byte(tt.pipeline)}) + assert.Equal(t, tt.expectedStatusCode, response.StatusCode) + if tt.expectedStatusCode == 403 { + assert.Equal(t, map[string]string{ + "error_code": "PERMISSION_DENIED", + "message": "User does not have CREATE TABLE on Schema 'main.src'.", + }, response.Body) + } + }) + } +} + func TestPipelineStartUpdate_HandlesNonExistentPipeline(t *testing.T) { workspace := NewFakeWorkspace("http://test", "dbapi123") From 9cb0a9cd74eee0e8f183c1d6cd54e51a77ff4254 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 22 Oct 2026 16:49:21 +0200 Subject: [PATCH 2/9] fix changelog --- .nextchanges/bundles/grants-before-dependents.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/grants-before-dependents.md b/.nextchanges/bundles/grants-before-dependents.md index 5070ca2d692..83b211658c0 100644 --- a/.nextchanges/bundles/grants-before-dependents.md +++ b/.nextchanges/bundles/grants-before-dependents.md @@ -1 +1 @@ -* Resources that reference another resource are now deployed after that resource's `grants`, fixing intermittent `User does not have CREATE TABLE on Schema` errors when a pipeline's `run_as` principal is granted access to its schema in the same bundle. +* Resources that reference another resource are now deployed after that resource's `grants`, fixing intermittent `User does not have CREATE TABLE on Schema` errors when a pipeline's `run_as` principal is granted access to its schema in the same bundle. ([#7142](https://github.com/databricks/cli/pull/7142)) From 8432652105a9cbdc50163d8dbaf2fcd23e4c6404 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 10 Nov 2026 09:28:46 +0100 Subject: [PATCH 3/9] Capture grants dependencies as lifecycle.depends_on A resource that depends on another resource, through a reference or through lifecycle.depends_on, now also depends on that resource's grants. The new CaptureGrantsDependencies mutator records this as a lifecycle.depends_on entry targeting the grants, e.g. schemas.src.grants, instead of adding edges to the deployment plan after the fact. A dependency that would create a cycle is skipped, e.g. when a schema grants privileges to the service principal of an app that refers to the schema. This requires lifecycle.depends_on to accept grants and permissions targets, which lands separately on the depends-on-grants branch. Co-authored-by: Isaac --- .../bundles/grants-before-dependents.md | 2 +- .../migrate/command/grants/out.new_state.json | 8 + .../grants/schemas/pipeline_run_as/output.txt | 27 ++- .../grants/schemas/pipeline_run_as/script | 4 + .../capture_grants_dependencies.go | 172 ++++++++++++++++++ .../capture_grants_dependencies_test.go | 137 ++++++++++++++ .../resourcemutator/resource_mutator.go | 4 + bundle/direct/bundle_plan.go | 61 ------- 8 files changed, 351 insertions(+), 64 deletions(-) create mode 100644 bundle/config/mutator/resourcemutator/capture_grants_dependencies.go create mode 100644 bundle/config/mutator/resourcemutator/capture_grants_dependencies_test.go diff --git a/.nextchanges/bundles/grants-before-dependents.md b/.nextchanges/bundles/grants-before-dependents.md index 83b211658c0..b3162ec8738 100644 --- a/.nextchanges/bundles/grants-before-dependents.md +++ b/.nextchanges/bundles/grants-before-dependents.md @@ -1 +1 @@ -* Resources that reference another resource are now deployed after that resource's `grants`, fixing intermittent `User does not have CREATE TABLE on Schema` errors when a pipeline's `run_as` principal is granted access to its schema in the same bundle. ([#7142](https://github.com/databricks/cli/pull/7142)) +* Resources that depend on another resource are now deployed after that resource's `grants`, through an implicit `lifecycle.depends_on` entry, fixing intermittent `User does not have CREATE TABLE on Schema` errors when a pipeline's `run_as` principal is granted access to its schema in the same bundle. ([#7142](https://github.com/databricks/cli/pull/7142)) diff --git a/acceptance/bundle/migrate/command/grants/out.new_state.json b/acceptance/bundle/migrate/command/grants/out.new_state.json index eb3b28ae467..4f68234acae 100644 --- a/acceptance/bundle/migrate/command/grants/out.new_state.json +++ b/acceptance/bundle/migrate/command/grants/out.new_state.json @@ -16,6 +16,10 @@ { "node": "resources.schemas.my_schema", "label": "${resources.schemas.my_schema.name}" + }, + { + "node": "resources.schemas.my_schema.grants", + "label": "depends_on" } ] }, @@ -83,6 +87,10 @@ { "node": "resources.schemas.my_schema", "label": "${resources.schemas.my_schema.name}" + }, + { + "node": "resources.schemas.my_schema.grants", + "label": "depends_on" } ] }, diff --git a/acceptance/bundle/resources/grants/schemas/pipeline_run_as/output.txt b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/output.txt index 9ec73822a86..4e15f2629e2 100644 --- a/acceptance/bundle/resources/grants/schemas/pipeline_run_as/output.txt +++ b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/output.txt @@ -1,4 +1,27 @@ +=== The grants of a referenced catalog or schema become an implicit lifecycle.depends_on +>>> [CLI] bundle validate -o json +{ + "catalogs": { + "pipeline_catalog": null + }, + "pipelines": { + "load": [ + { + "resource": "schemas.src.grants" + } + ] + }, + "schemas": { + "src": [ + { + "resource": "catalogs.pipeline_catalog.grants" + } + ] + } +} + +=== The plan orders dependents after the grants >>> [CLI] bundle plan -o json { "resources.catalogs.pipeline_catalog": null, @@ -19,7 +42,7 @@ }, { "node": "resources.schemas.src.grants", - "label": "${resources.schemas.src.catalog_name}" + "label": "depends_on" } ], "resources.schemas.src": [ @@ -29,7 +52,7 @@ }, { "node": "resources.catalogs.pipeline_catalog.grants", - "label": "${resources.catalogs.pipeline_catalog.name}" + "label": "depends_on" } ], "resources.schemas.src.grants": [ diff --git a/acceptance/bundle/resources/grants/schemas/pipeline_run_as/script b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/script index 9ce8ea179b3..392803e4258 100644 --- a/acceptance/bundle/resources/grants/schemas/pipeline_run_as/script +++ b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/script @@ -10,6 +10,10 @@ cleanup() { } trap cleanup EXIT +title "The grants of a referenced catalog or schema become an implicit lifecycle.depends_on" +trace $CLI bundle validate -o json | jq '.resources | map_values(map_values(.lifecycle.depends_on))' + +title "The plan orders dependents after the grants" trace $CLI bundle plan -o json | jq '.plan | map_values(.depends_on)' trace $CLI bundle deploy diff --git a/bundle/config/mutator/resourcemutator/capture_grants_dependencies.go b/bundle/config/mutator/resourcemutator/capture_grants_dependencies.go new file mode 100644 index 00000000000..00008db1360 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/capture_grants_dependencies.go @@ -0,0 +1,172 @@ +package resourcemutator + +import ( + "context" + "fmt" + "maps" + "reflect" + "slices" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" +) + +type captureGrantsDependencies struct{} + +// CaptureGrantsDependencies makes a resource that depends on another resource also depend on +// that resource's grants, because creating some resources requires the grants to be in place: +// a pipeline cannot be created before its run_as principal has CREATE TABLE on the target +// schema. The dependency is added to lifecycle.depends_on, and skipped when it would create a +// cycle, e.g. when a schema grants privileges to the service principal of an app that refers +// to the schema. +func CaptureGrantsDependencies() bundle.Mutator { + return &captureGrantsDependencies{} +} + +func (m *captureGrantsDependencies) Name() string { + return "CaptureGrantsDependencies" +} + +func (m *captureGrantsDependencies) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + graph, err := dependencyGraph(b) + if err != nil { + return diag.FromErr(err) + } + + // Resources are visited in a fixed order because an added dependency can rule out a later + // one: with two of them forming a cycle together, the same one must be skipped on every run. + added := make(map[string][]string) + for _, node := range slices.Sorted(maps.Keys(graph)) { + // Grants and permissions have no lifecycle. + if strings.Contains(config.GetResourceTypeFromKey(node), ".") { + continue + } + + // The grants appended below are not visited: range evaluates the slice once. + for _, dep := range graph[node] { + grants := dep + ".grants" + if _, ok := graph[grants]; !ok || slices.Contains(graph[node], grants) || reaches(graph, grants, node) { + continue + } + graph[node] = append(graph[node], grants) + added[node] = append(added[node], grants) + } + } + + resources := dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()) + err = b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(root, resources, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + grants := added[p.String()] + if len(grants) == 0 { + return v, nil + } + return appendDependsOn(v, grants) + }) + }) + return diag.FromErr(err) +} + +// dependencyGraph returns, for every resource and its grants or permissions, the nodes it depends +// on: those it refers to and those in its lifecycle.depends_on. Nodes are keyed as in the deployment +// plan, e.g. "resources.schemas.foo.grants"; a node without dependencies maps to nil. +func dependencyGraph(b *bundle.Bundle) (map[string][]string, error) { + graph := make(map[string][]string) + + err := dyn.WalkReadOnly(b.Config.Value(), func(p dyn.Path, v dyn.Value) error { + node, resourceType := config.GetNodeAndType(p) + if resourceType == "" { + return nil + } + + src := node.String() + if len(p) == len(node) { + // Grants and permissions without entries are not deployed, so nothing can depend on them. + if seq, ok := v.AsSequence(); len(node) == 4 && (!ok || len(seq) == 0) { + return nil + } + if _, ok := graph[src]; !ok { + graph[src] = nil + } + return nil + } + + ref, ok := dynvar.NewRef(v) + if !ok { + return nil + } + for _, r := range ref.References() { + path, err := dyn.NewPathFromString(r) + if err != nil { + return fmt.Errorf("parsing %q: %w", r, err) + } + if target, targetType := config.GetNodeAndType(path); targetType != "" { + graph[src] = append(graph[src], target.String()) + } + } + return nil + }) + if err != nil { + return nil, err + } + + for _, group := range b.Config.Resources.AllResources() { + for key, resource := range group.Resources { + // Python-generated configs can contain null resource entries. + if reflect.ValueOf(resource).IsNil() { + continue + } + src := "resources." + group.Description.PluralName + "." + key + for _, dep := range resource.GetLifecycle().GetDependsOn() { + // An entry that does not resolve is reported by validate.LifecycleDependsOn. + if target, err := b.Config.Resources.ResolveDependsOn(dep.Resource); err == nil { + graph[src] = append(graph[src], target) + } + } + } + } + return graph, nil +} + +// reaches reports whether from is to or depends on it, directly or through other nodes. +func reaches(graph map[string][]string, from, to string) bool { + visited := make(map[string]bool) + stack := []string{from} + for len(stack) > 0 { + n := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if n == to { + return true + } + if visited[n] { + continue + } + visited[n] = true + stack = append(stack, graph[n]...) + } + return false +} + +// appendDependsOn appends lifecycle.depends_on entries for the given nodes to a resource. +func appendDependsOn(resource dyn.Value, nodes []string) (dyn.Value, error) { + lifecycle := dyn.GetValue(resource, "lifecycle") + if lifecycle.Kind() != dyn.KindMap { + lifecycle = dyn.V(dyn.NewMapping()) + } + + entries, _ := dyn.GetValue(lifecycle, "depends_on").AsSequence() + for _, node := range nodes { + entries = append(entries, dyn.V(map[string]dyn.Value{ + "resource": dyn.V(strings.TrimPrefix(node, "resources.")), + })) + } + + lifecycle, err := dyn.Set(lifecycle, "depends_on", dyn.V(entries)) + if err != nil { + return dyn.InvalidValue, err + } + return dyn.Set(resource, "lifecycle", lifecycle) +} diff --git a/bundle/config/mutator/resourcemutator/capture_grants_dependencies_test.go b/bundle/config/mutator/resourcemutator/capture_grants_dependencies_test.go new file mode 100644 index 00000000000..0cc50f327ad --- /dev/null +++ b/bundle/config/mutator/resourcemutator/capture_grants_dependencies_test.go @@ -0,0 +1,137 @@ +package resourcemutator + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCaptureGrantsDependencies(t *testing.T) { + grants := func(principal string) []catalog.PrivilegeAssignment { + return []catalog.PrivilegeAssignment{{Principal: principal, Privileges: []catalog.Privilege{catalog.PrivilegeCreateTable}}} + } + schema := func(grants []catalog.PrivilegeAssignment) *resources.Schema { + return &resources.Schema{CatalogName: "main", Name: "src", Grants: grants} + } + // A pipeline that refers to the schema. + pipeline := &resources.Pipeline{ + Catalog: "${resources.schemas.src.catalog_name}", + Schema: "${resources.schemas.src.name}", + } + // An app that refers to the schema. + app := &resources.App{Name: "app", Description: "${resources.schemas.src.name}"} + + tests := []struct { + name string + resources config.Resources + expected map[string][]resources.DependsOn + }{ + { + name: "resource that refers to a resource with grants depends on the grants", + resources: config.Resources{ + Schemas: map[string]*resources.Schema{"src": schema(grants("sp"))}, + Pipelines: map[string]*resources.Pipeline{"load": pipeline}, + }, + expected: map[string][]resources.DependsOn{ + "pipelines.load": {{Resource: "schemas.src.grants"}}, + }, + }, + { + name: "resource without grants", + resources: config.Resources{ + Schemas: map[string]*resources.Schema{"src": schema(nil)}, + Pipelines: map[string]*resources.Pipeline{"load": pipeline}, + }, + expected: map[string][]resources.DependsOn{}, + }, + { + name: "each resource depends on the grants of the resource it refers to", + resources: config.Resources{ + Catalogs: map[string]*resources.Catalog{"main": {CreateCatalog: catalog.CreateCatalog{Name: "main"}, Grants: grants("sp")}}, + Schemas: map[string]*resources.Schema{"src": { + CreateSchema: catalog.CreateSchema{CatalogName: "${resources.catalogs.main.name}", Name: "src"}, + Grants: grants("sp"), + }}, + Pipelines: map[string]*resources.Pipeline{"load": pipeline}, + }, + expected: map[string][]resources.DependsOn{ + "schemas.src": {{Resource: "catalogs.main.grants"}}, + "pipelines.load": {{Resource: "schemas.src.grants"}}, + }, + }, + { + name: "grants that depend on the referring resource", + resources: config.Resources{ + Schemas: map[string]*resources.Schema{"src": schema(grants("${resources.apps.app.service_principal_client_id}"))}, + Apps: map[string]*resources.App{"app": app}, + Pipelines: map[string]*resources.Pipeline{"load": pipeline}, + }, + expected: map[string][]resources.DependsOn{ + "pipelines.load": {{Resource: "schemas.src.grants"}}, + }, + }, + { + name: "grants that transitively depend on the referring resource", + resources: config.Resources{ + Schemas: map[string]*resources.Schema{"src": schema(grants("${resources.jobs.job.run_as.service_principal_name}"))}, + Jobs: map[string]*resources.Job{"job": {JobSettings: jobs.JobSettings{Name: "${resources.apps.app.name}"}}}, + Apps: map[string]*resources.App{"app": app}, + }, + expected: map[string][]resources.DependsOn{}, + }, + { + name: "lifecycle.depends_on on a resource implies its grants", + resources: config.Resources{ + Schemas: map[string]*resources.Schema{"src": schema(grants("sp"))}, + Jobs: map[string]*resources.Job{"job": { + BaseResource: resources.BaseResource{Lifecycle: resources.Lifecycle{DependsOn: []resources.DependsOn{{Resource: "schemas.src"}}}}, + JobSettings: jobs.JobSettings{Name: "job"}, + }}, + }, + expected: map[string][]resources.DependsOn{ + "jobs.job": {{Resource: "schemas.src"}, {Resource: "schemas.src.grants"}}, + }, + }, + { + name: "grants already in lifecycle.depends_on", + resources: config.Resources{ + Schemas: map[string]*resources.Schema{"src": schema(grants("sp"))}, + Pipelines: map[string]*resources.Pipeline{"load": { + BaseResource: resources.BaseResource{Lifecycle: resources.Lifecycle{DependsOn: []resources.DependsOn{{Resource: "schemas.src.grants"}}}}, + CreatePipeline: pipeline.CreatePipeline, + }}, + }, + expected: map[string][]resources.DependsOn{ + "pipelines.load": {{Resource: "schemas.src.grants"}}, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + b := &bundle.Bundle{Config: config.Root{Resources: tc.resources}} + + // Normalization runs more than once, so applying the mutator again must not add anything. + for range 2 { + diags := bundle.Apply(t.Context(), b, CaptureGrantsDependencies()) + require.NoError(t, diags.Error()) + + actual := make(map[string][]resources.DependsOn) + for _, group := range b.Config.Resources.AllResources() { + for key, resource := range group.Resources { + if deps := resource.GetLifecycle().GetDependsOn(); len(deps) > 0 { + actual[group.Description.PluralName+"."+key] = deps + } + } + } + assert.Equal(t, tc.expected, actual) + } + }) + } +} diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index e00444cc773..35d8d9bbe8f 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -183,6 +183,10 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Translates implicit schema and catalog references across all UC resources to explicit syntax to capture dependencies CaptureUCDependencies(), + // Reads (dynamic): resources.*.** (searches for ${resources.*} references), resources.*.*.grants, resources.*.*.lifecycle.depends_on + // Updates (dynamic): resources.*.*.lifecycle.depends_on (adds the grants of the resources a resource depends on) + CaptureGrantsDependencies(), + // Reads (dynamic): resources.dashboards.*.file_path // Updates (dynamic): resources.dashboards.*.serialized_dashboard // Drops (dynamic): resources.dashboards.*.file_path diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index f2de4ad918e..e1b8f93bebb 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -1795,67 +1795,6 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root return p, nil } -func compareDependsOn(a, b deployplan.DependsOnEntry) int { - if a.Node != b.Node { - return strings.Compare(a.Node, b.Node) - } - return strings.Compare(a.Label, b.Label) -} - -// addGrantsDependencies makes resources that refer to another resource also depend on that -// resource's grants, because creating some resources requires the grants to be in place: a -// pipeline cannot be created before its run_as principal has CREATE TABLE on the target schema. -// A grants dependency that would create a cycle is skipped, e.g. when a schema grants privileges -// to the service principal of an app that refers to the schema. -func addGrantsDependencies(plan *deployplan.Plan) { - for _, node := range slices.Sorted(maps.Keys(plan.Plan)) { - entry := plan.Plan[node] - - seen := make(map[string]bool, len(entry.DependsOn)) - for _, dep := range entry.DependsOn { - seen[dep.Node] = true - } - - var grantsDeps []deployplan.DependsOnEntry - for _, dep := range entry.DependsOn { - grantsNode := dep.Node + ".grants" - if seen[grantsNode] || plan.Plan[grantsNode] == nil || dependsOnTransitively(plan, grantsNode, node) { - continue - } - seen[grantsNode] = true - grantsDeps = append(grantsDeps, deployplan.DependsOnEntry{Node: grantsNode, Label: dep.Label}) - } - - if len(grantsDeps) > 0 { - entry.DependsOn = append(entry.DependsOn, grantsDeps...) - slices.SortFunc(entry.DependsOn, compareDependsOn) - } - } -} - -// dependsOnTransitively reports whether node is target or depends on it, directly or through other nodes. -func dependsOnTransitively(plan *deployplan.Plan, node, target string) bool { - visited := make(map[string]bool) - stack := []string{node} - for len(stack) > 0 { - n := stack[len(stack)-1] - stack = stack[:len(stack)-1] - if n == target { - return true - } - // A dependency on a node missing from the plan is reported by makeGraph. - entry := plan.Plan[n] - if visited[n] || entry == nil { - continue - } - visited[n] = true - for _, dep := range entry.DependsOn { - stack = append(stack, dep.Node) - } - } - return false -} - // ExtractReferences extracts variable references from the config subtree rooted at node, // keeping only those whose field path exists in stateType (references in input-only or // bundle:"readonly" fields, such as volumes' computed volume_path, are skipped). From 5f8f9543023c0e98b34d7d1144e26597a772c620 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 11 Nov 2026 12:05:25 +0100 Subject: [PATCH 4/9] Remove leftover addGrantsDependencies call from direct plan The previous commit moved grants dependencies into the CaptureGrantsDependencies mutator, which records them as lifecycle.depends_on entries. It deleted the addGrantsDependencies helper but left its call in makePlan, which broke the build of bundle/direct and every job that compiles it. Remove the call so the plan relies only on the mutator output. Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index e1b8f93bebb..43a58291ab2 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -1771,8 +1771,6 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root p.Plan[node] = &e } - addGrantsDependencies(p) - // lifecycle.depends_on only orders resources: a target that is not in the plan // (e.g. skipped as empty) needs no ordering. for _, entry := range p.Plan { From d7beef682d05bb88a976c3cc99443b6709858cbe Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Sun, 15 Nov 2026 18:06:04 +0000 Subject: [PATCH 5/9] Rename CaptureGrantsDependencies to CapturePipelineDependencies The mutator is about to be limited to pipelines, so name it after them. This commit only renames the mutator and its file; the behaviour changes next. Co-authored-by: Isaac --- ...dencies.go => capture_pipeline_dependencies.go} | 14 +++++++------- ...st.go => capture_pipeline_dependencies_test.go} | 4 ++-- .../mutator/resourcemutator/resource_mutator.go | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) rename bundle/config/mutator/resourcemutator/{capture_grants_dependencies.go => capture_pipeline_dependencies.go} (92%) rename bundle/config/mutator/resourcemutator/{capture_grants_dependencies_test.go => capture_pipeline_dependencies_test.go} (97%) diff --git a/bundle/config/mutator/resourcemutator/capture_grants_dependencies.go b/bundle/config/mutator/resourcemutator/capture_pipeline_dependencies.go similarity index 92% rename from bundle/config/mutator/resourcemutator/capture_grants_dependencies.go rename to bundle/config/mutator/resourcemutator/capture_pipeline_dependencies.go index 00008db1360..3b175052b9a 100644 --- a/bundle/config/mutator/resourcemutator/capture_grants_dependencies.go +++ b/bundle/config/mutator/resourcemutator/capture_pipeline_dependencies.go @@ -15,23 +15,23 @@ import ( "github.com/databricks/cli/libs/dyn/dynvar" ) -type captureGrantsDependencies struct{} +type capturePipelineDependencies struct{} -// CaptureGrantsDependencies makes a resource that depends on another resource also depend on +// CapturePipelineDependencies makes a resource that depends on another resource also depend on // that resource's grants, because creating some resources requires the grants to be in place: // a pipeline cannot be created before its run_as principal has CREATE TABLE on the target // schema. The dependency is added to lifecycle.depends_on, and skipped when it would create a // cycle, e.g. when a schema grants privileges to the service principal of an app that refers // to the schema. -func CaptureGrantsDependencies() bundle.Mutator { - return &captureGrantsDependencies{} +func CapturePipelineDependencies() bundle.Mutator { + return &capturePipelineDependencies{} } -func (m *captureGrantsDependencies) Name() string { - return "CaptureGrantsDependencies" +func (m *capturePipelineDependencies) Name() string { + return "CapturePipelineDependencies" } -func (m *captureGrantsDependencies) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *capturePipelineDependencies) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { graph, err := dependencyGraph(b) if err != nil { return diag.FromErr(err) diff --git a/bundle/config/mutator/resourcemutator/capture_grants_dependencies_test.go b/bundle/config/mutator/resourcemutator/capture_pipeline_dependencies_test.go similarity index 97% rename from bundle/config/mutator/resourcemutator/capture_grants_dependencies_test.go rename to bundle/config/mutator/resourcemutator/capture_pipeline_dependencies_test.go index 0cc50f327ad..a7a4eb08232 100644 --- a/bundle/config/mutator/resourcemutator/capture_grants_dependencies_test.go +++ b/bundle/config/mutator/resourcemutator/capture_pipeline_dependencies_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestCaptureGrantsDependencies(t *testing.T) { +func TestCapturePipelineDependencies(t *testing.T) { grants := func(principal string) []catalog.PrivilegeAssignment { return []catalog.PrivilegeAssignment{{Principal: principal, Privileges: []catalog.Privilege{catalog.PrivilegeCreateTable}}} } @@ -119,7 +119,7 @@ func TestCaptureGrantsDependencies(t *testing.T) { // Normalization runs more than once, so applying the mutator again must not add anything. for range 2 { - diags := bundle.Apply(t.Context(), b, CaptureGrantsDependencies()) + diags := bundle.Apply(t.Context(), b, CapturePipelineDependencies()) require.NoError(t, diags.Error()) actual := make(map[string][]resources.DependsOn) diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 35d8d9bbe8f..2e1679eab34 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -185,7 +185,7 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Reads (dynamic): resources.*.** (searches for ${resources.*} references), resources.*.*.grants, resources.*.*.lifecycle.depends_on // Updates (dynamic): resources.*.*.lifecycle.depends_on (adds the grants of the resources a resource depends on) - CaptureGrantsDependencies(), + CapturePipelineDependencies(), // Reads (dynamic): resources.dashboards.*.file_path // Updates (dynamic): resources.dashboards.*.serialized_dashboard From c11ba4a10105abd58beec0e36240038572eb72c9 Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Sun, 15 Nov 2026 18:08:49 +0000 Subject: [PATCH 6/9] Only add grants and permissions depends_on for pipelines A pipeline cannot be created until the privileges it depends on are in place, for example its run_as principal's CREATE TABLE on the schema it writes to. Limit the implicit lifecycle.depends_on entries to pipelines, and make each pipeline wait for the grants and permissions of the resources it refers to, directly or through lifecycle.depends_on, when those exist. Entries are written through the typed config. A dependency that would create a cycle is still skipped, and the cycle check considers the whole graph. Co-authored-by: Isaac --- .../migrate/command/grants/out.new_state.json | 8 -- .../capture_pipeline_dependencies.go | 71 +++-------- .../capture_pipeline_dependencies_test.go | 117 ++++++++++++++---- .../resourcemutator/resource_mutator.go | 4 +- 4 files changed, 109 insertions(+), 91 deletions(-) diff --git a/acceptance/bundle/migrate/command/grants/out.new_state.json b/acceptance/bundle/migrate/command/grants/out.new_state.json index 4f68234acae..eb3b28ae467 100644 --- a/acceptance/bundle/migrate/command/grants/out.new_state.json +++ b/acceptance/bundle/migrate/command/grants/out.new_state.json @@ -16,10 +16,6 @@ { "node": "resources.schemas.my_schema", "label": "${resources.schemas.my_schema.name}" - }, - { - "node": "resources.schemas.my_schema.grants", - "label": "depends_on" } ] }, @@ -87,10 +83,6 @@ { "node": "resources.schemas.my_schema", "label": "${resources.schemas.my_schema.name}" - }, - { - "node": "resources.schemas.my_schema.grants", - "label": "depends_on" } ] }, diff --git a/bundle/config/mutator/resourcemutator/capture_pipeline_dependencies.go b/bundle/config/mutator/resourcemutator/capture_pipeline_dependencies.go index 3b175052b9a..98408eca92c 100644 --- a/bundle/config/mutator/resourcemutator/capture_pipeline_dependencies.go +++ b/bundle/config/mutator/resourcemutator/capture_pipeline_dependencies.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" @@ -17,12 +18,10 @@ import ( type capturePipelineDependencies struct{} -// CapturePipelineDependencies makes a resource that depends on another resource also depend on -// that resource's grants, because creating some resources requires the grants to be in place: -// a pipeline cannot be created before its run_as principal has CREATE TABLE on the target -// schema. The dependency is added to lifecycle.depends_on, and skipped when it would create a -// cycle, e.g. when a schema grants privileges to the service principal of an app that refers -// to the schema. +// CapturePipelineDependencies makes a pipeline wait for the grants and permissions of the resources +// it refers to, directly or through lifecycle.depends_on, so a pipeline is not created before the +// privileges it depends on are in place. A dependency that would create a cycle is skipped, e.g. +// when an app refers to the pipeline and its service principal is granted on the pipeline's schema. func CapturePipelineDependencies() bundle.Mutator { return &capturePipelineDependencies{} } @@ -37,37 +36,22 @@ func (m *capturePipelineDependencies) Apply(ctx context.Context, b *bundle.Bundl return diag.FromErr(err) } - // Resources are visited in a fixed order because an added dependency can rule out a later - // one: with two of them forming a cycle together, the same one must be skipped on every run. - added := make(map[string][]string) - for _, node := range slices.Sorted(maps.Keys(graph)) { - // Grants and permissions have no lifecycle. - if strings.Contains(config.GetResourceTypeFromKey(node), ".") { - continue - } - - // The grants appended below are not visited: range evaluates the slice once. + // Pipelines are visited in a fixed order, so the same cycle-forming dependency is skipped on every run. + for _, key := range slices.Sorted(maps.Keys(b.Config.Resources.Pipelines)) { + p := b.Config.Resources.Pipelines[key] + node := "resources.pipelines." + key for _, dep := range graph[node] { - grants := dep + ".grants" - if _, ok := graph[grants]; !ok || slices.Contains(graph[node], grants) || reaches(graph, grants, node) { - continue + for _, sub := range []string{".grants", ".permissions"} { + target := dep + sub + if _, ok := graph[target]; !ok || slices.Contains(graph[node], target) || reaches(graph, target, node) { + continue + } + graph[node] = append(graph[node], target) + p.Lifecycle.DependsOn = append(p.Lifecycle.DependsOn, resources.DependsOn{Resource: strings.TrimPrefix(target, "resources.")}) } - graph[node] = append(graph[node], grants) - added[node] = append(added[node], grants) } } - - resources := dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()) - err = b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(root, resources, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - grants := added[p.String()] - if len(grants) == 0 { - return v, nil - } - return appendDependsOn(v, grants) - }) - }) - return diag.FromErr(err) + return nil } // dependencyGraph returns, for every resource and its grants or permissions, the nodes it depends @@ -149,24 +133,3 @@ func reaches(graph map[string][]string, from, to string) bool { } return false } - -// appendDependsOn appends lifecycle.depends_on entries for the given nodes to a resource. -func appendDependsOn(resource dyn.Value, nodes []string) (dyn.Value, error) { - lifecycle := dyn.GetValue(resource, "lifecycle") - if lifecycle.Kind() != dyn.KindMap { - lifecycle = dyn.V(dyn.NewMapping()) - } - - entries, _ := dyn.GetValue(lifecycle, "depends_on").AsSequence() - for _, node := range nodes { - entries = append(entries, dyn.V(map[string]dyn.Value{ - "resource": dyn.V(strings.TrimPrefix(node, "resources.")), - })) - } - - lifecycle, err := dyn.Set(lifecycle, "depends_on", dyn.V(entries)) - if err != nil { - return dyn.InvalidValue, err - } - return dyn.Set(resource, "lifecycle", lifecycle) -} diff --git a/bundle/config/mutator/resourcemutator/capture_pipeline_dependencies_test.go b/bundle/config/mutator/resourcemutator/capture_pipeline_dependencies_test.go index a7a4eb08232..d51fd9925e6 100644 --- a/bundle/config/mutator/resourcemutator/capture_pipeline_dependencies_test.go +++ b/bundle/config/mutator/resourcemutator/capture_pipeline_dependencies_test.go @@ -1,6 +1,7 @@ package resourcemutator import ( + "reflect" "testing" "github.com/databricks/cli/bundle" @@ -8,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -19,13 +21,21 @@ func TestCapturePipelineDependencies(t *testing.T) { schema := func(grants []catalog.PrivilegeAssignment) *resources.Schema { return &resources.Schema{CatalogName: "main", Name: "src", Grants: grants} } - // A pipeline that refers to the schema. - pipeline := &resources.Pipeline{ - Catalog: "${resources.schemas.src.catalog_name}", - Schema: "${resources.schemas.src.name}", + // A pipeline that refers to the schema. A constructor, because the mutator writes to the + // pipeline and a shared pointer would leak lifecycle.depends_on between cases. + pipeline := func() *resources.Pipeline { + return &resources.Pipeline{CreatePipeline: pipelines.CreatePipeline{ + Catalog: "${resources.schemas.src.catalog_name}", + Schema: "${resources.schemas.src.name}", + }} + } + // A job with permissions, referred to by name. + job := func(permissions []resources.JobPermission) *resources.Job { + return &resources.Job{ + JobSettings: jobs.JobSettings{Name: "job"}, + Permissions: permissions, + } } - // An app that refers to the schema. - app := &resources.App{Name: "app", Description: "${resources.schemas.src.name}"} tests := []struct { name string @@ -33,69 +43,87 @@ func TestCapturePipelineDependencies(t *testing.T) { expected map[string][]resources.DependsOn }{ { - name: "resource that refers to a resource with grants depends on the grants", + name: "pipeline that refers to a resource with grants depends on the grants", resources: config.Resources{ Schemas: map[string]*resources.Schema{"src": schema(grants("sp"))}, - Pipelines: map[string]*resources.Pipeline{"load": pipeline}, + Pipelines: map[string]*resources.Pipeline{"load": pipeline()}, }, expected: map[string][]resources.DependsOn{ "pipelines.load": {{Resource: "schemas.src.grants"}}, }, }, { - name: "resource without grants", + name: "empty grants and permissions add nothing", resources: config.Resources{ - Schemas: map[string]*resources.Schema{"src": schema(nil)}, - Pipelines: map[string]*resources.Pipeline{"load": pipeline}, + Schemas: map[string]*resources.Schema{"src": schema([]catalog.PrivilegeAssignment{})}, + Jobs: map[string]*resources.Job{"job": job([]resources.JobPermission{})}, + Pipelines: map[string]*resources.Pipeline{"load": {CreatePipeline: pipelines.CreatePipeline{Name: "${resources.jobs.job.name}", Catalog: "${resources.schemas.src.catalog_name}", Schema: "${resources.schemas.src.name}"}}}, }, expected: map[string][]resources.DependsOn{}, }, { - name: "each resource depends on the grants of the resource it refers to", + name: "null pipeline entry does not panic and other pipelines are still processed", + resources: config.Resources{ + Schemas: map[string]*resources.Schema{"src": schema(grants("sp"))}, + Pipelines: map[string]*resources.Pipeline{"empty": nil, "load": pipeline()}, + }, + expected: map[string][]resources.DependsOn{ + "pipelines.load": {{Resource: "schemas.src.grants"}}, + }, + }, + { + name: "pipeline waits for the schema's grants, not its catalog's", resources: config.Resources{ Catalogs: map[string]*resources.Catalog{"main": {CreateCatalog: catalog.CreateCatalog{Name: "main"}, Grants: grants("sp")}}, Schemas: map[string]*resources.Schema{"src": { CreateSchema: catalog.CreateSchema{CatalogName: "${resources.catalogs.main.name}", Name: "src"}, Grants: grants("sp"), }}, - Pipelines: map[string]*resources.Pipeline{"load": pipeline}, + Pipelines: map[string]*resources.Pipeline{"load": pipeline()}, }, expected: map[string][]resources.DependsOn{ - "schemas.src": {{Resource: "catalogs.main.grants"}}, "pipelines.load": {{Resource: "schemas.src.grants"}}, }, }, { - name: "grants that depend on the referring resource", + name: "non-pipeline resource that refers to a resource with grants gets nothing", + resources: config.Resources{ + Schemas: map[string]*resources.Schema{"src": schema(grants("sp"))}, + Jobs: map[string]*resources.Job{"job": {JobSettings: jobs.JobSettings{Name: "${resources.schemas.src.name}"}}}, + }, + expected: map[string][]resources.DependsOn{}, + }, + { + name: "grants that refer to an app that does not refer to the pipeline are added", resources: config.Resources{ Schemas: map[string]*resources.Schema{"src": schema(grants("${resources.apps.app.service_principal_client_id}"))}, - Apps: map[string]*resources.App{"app": app}, - Pipelines: map[string]*resources.Pipeline{"load": pipeline}, + Apps: map[string]*resources.App{"app": {Name: "app", Description: "${resources.schemas.src.name}"}}, + Pipelines: map[string]*resources.Pipeline{"load": pipeline()}, }, expected: map[string][]resources.DependsOn{ "pipelines.load": {{Resource: "schemas.src.grants"}}, }, }, { - name: "grants that transitively depend on the referring resource", + name: "grants that depend on an app that refers to the pipeline are skipped", resources: config.Resources{ - Schemas: map[string]*resources.Schema{"src": schema(grants("${resources.jobs.job.run_as.service_principal_name}"))}, - Jobs: map[string]*resources.Job{"job": {JobSettings: jobs.JobSettings{Name: "${resources.apps.app.name}"}}}, - Apps: map[string]*resources.App{"app": app}, + Schemas: map[string]*resources.Schema{"src": schema(grants("${resources.apps.app.service_principal_client_id}"))}, + Apps: map[string]*resources.App{"app": {Name: "app", Description: "${resources.pipelines.load.name}"}}, + Pipelines: map[string]*resources.Pipeline{"load": pipeline()}, }, expected: map[string][]resources.DependsOn{}, }, { - name: "lifecycle.depends_on on a resource implies its grants", + name: "lifecycle.depends_on on a pipeline implies the grants of the resource", resources: config.Resources{ Schemas: map[string]*resources.Schema{"src": schema(grants("sp"))}, - Jobs: map[string]*resources.Job{"job": { - BaseResource: resources.BaseResource{Lifecycle: resources.Lifecycle{DependsOn: []resources.DependsOn{{Resource: "schemas.src"}}}}, - JobSettings: jobs.JobSettings{Name: "job"}, + Pipelines: map[string]*resources.Pipeline{"load": { + BaseResource: resources.BaseResource{Lifecycle: resources.Lifecycle{DependsOn: []resources.DependsOn{{Resource: "schemas.src"}}}}, + CreatePipeline: pipeline().CreatePipeline, }}, }, expected: map[string][]resources.DependsOn{ - "jobs.job": {{Resource: "schemas.src"}, {Resource: "schemas.src.grants"}}, + "pipelines.load": {{Resource: "schemas.src"}, {Resource: "schemas.src.grants"}}, }, }, { @@ -104,13 +132,45 @@ func TestCapturePipelineDependencies(t *testing.T) { Schemas: map[string]*resources.Schema{"src": schema(grants("sp"))}, Pipelines: map[string]*resources.Pipeline{"load": { BaseResource: resources.BaseResource{Lifecycle: resources.Lifecycle{DependsOn: []resources.DependsOn{{Resource: "schemas.src.grants"}}}}, - CreatePipeline: pipeline.CreatePipeline, + CreatePipeline: pipeline().CreatePipeline, }}, }, expected: map[string][]resources.DependsOn{ "pipelines.load": {{Resource: "schemas.src.grants"}}, }, }, + { + name: "pipeline depends on the permissions of a resource it refers to", + resources: config.Resources{ + Jobs: map[string]*resources.Job{"job": job([]resources.JobPermission{{Level: "CAN_VIEW", GroupName: "users"}})}, + Pipelines: map[string]*resources.Pipeline{"load": { + CreatePipeline: pipelines.CreatePipeline{Name: "${resources.jobs.job.name}"}, + }}, + }, + expected: map[string][]resources.DependsOn{ + "pipelines.load": {{Resource: "jobs.job.permissions"}}, + }, + }, + { + name: "non-pipeline resource that refers to a resource with permissions gets nothing", + resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": job([]resources.JobPermission{{Level: "CAN_VIEW", GroupName: "users"}}), + "reader": {JobSettings: jobs.JobSettings{Name: "${resources.jobs.job.name}"}}, + }, + }, + expected: map[string][]resources.DependsOn{}, + }, + { + name: "permissions that depend on the referring pipeline are skipped", + resources: config.Resources{ + Jobs: map[string]*resources.Job{"job": job([]resources.JobPermission{{Level: "CAN_VIEW", UserName: "${resources.pipelines.load.name}"}})}, + Pipelines: map[string]*resources.Pipeline{"load": { + CreatePipeline: pipelines.CreatePipeline{Name: "${resources.jobs.job.name}"}, + }}, + }, + expected: map[string][]resources.DependsOn{}, + }, } for _, tc := range tests { @@ -125,6 +185,9 @@ func TestCapturePipelineDependencies(t *testing.T) { actual := make(map[string][]resources.DependsOn) for _, group := range b.Config.Resources.AllResources() { for key, resource := range group.Resources { + if reflect.ValueOf(resource).IsNil() { + continue + } if deps := resource.GetLifecycle().GetDependsOn(); len(deps) > 0 { actual[group.Description.PluralName+"."+key] = deps } diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 2e1679eab34..16a997ebf24 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -183,8 +183,8 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Translates implicit schema and catalog references across all UC resources to explicit syntax to capture dependencies CaptureUCDependencies(), - // Reads (dynamic): resources.*.** (searches for ${resources.*} references), resources.*.*.grants, resources.*.*.lifecycle.depends_on - // Updates (dynamic): resources.*.*.lifecycle.depends_on (adds the grants of the resources a resource depends on) + // Reads (dynamic and typed): resources.*.** (searches for ${resources.*} references), resources.*.*.{grants,permissions,lifecycle.depends_on} + // Updates (typed): resources.pipelines.*.lifecycle.depends_on (adds the grants and permissions of the resources a pipeline depends on) CapturePipelineDependencies(), // Reads (dynamic): resources.dashboards.*.file_path From bca6c4bc469b33f8d5a5236656acdb8369a973ce Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Sun, 15 Nov 2026 18:08:50 +0000 Subject: [PATCH 7/9] Cover pipeline depends_on in acceptance tests and changelog The pipeline_run_as test checks the implicit depends_on in validate and plan and the schema request order on deploy. A new permissions test checks that a pipeline waits for the permissions of a job it refers to. The migrate test is restored to origin/main, since only pipelines get implicit entries now. Co-authored-by: Isaac --- .../bundles/grants-before-dependents.md | 2 +- .../grants/schemas/pipeline_run_as/output.txt | 18 ++++----------- .../grants/schemas/pipeline_run_as/script | 13 ++++++++--- .../pipelines/depends_on/databricks.yml | 15 ++++++++++++ .../pipelines/depends_on/out.test.toml | 2 ++ .../pipelines/depends_on/output.txt | 23 +++++++++++++++++++ .../permissions/pipelines/depends_on/script | 2 ++ 7 files changed, 58 insertions(+), 17 deletions(-) create mode 100644 acceptance/bundle/resources/permissions/pipelines/depends_on/databricks.yml create mode 100644 acceptance/bundle/resources/permissions/pipelines/depends_on/out.test.toml create mode 100644 acceptance/bundle/resources/permissions/pipelines/depends_on/output.txt create mode 100644 acceptance/bundle/resources/permissions/pipelines/depends_on/script diff --git a/.nextchanges/bundles/grants-before-dependents.md b/.nextchanges/bundles/grants-before-dependents.md index b3162ec8738..6e4386567e4 100644 --- a/.nextchanges/bundles/grants-before-dependents.md +++ b/.nextchanges/bundles/grants-before-dependents.md @@ -1 +1 @@ -* Resources that depend on another resource are now deployed after that resource's `grants`, through an implicit `lifecycle.depends_on` entry, fixing intermittent `User does not have CREATE TABLE on Schema` errors when a pipeline's `run_as` principal is granted access to its schema in the same bundle. ([#7142](https://github.com/databricks/cli/pull/7142)) +* Pipelines now wait for the grants and permissions of the resources they refer to or list in `lifecycle.depends_on`, so a pipeline whose run_as principal is granted access to its schema in the same bundle no longer fails with `User does not have CREATE TABLE on Schema`. diff --git a/acceptance/bundle/resources/grants/schemas/pipeline_run_as/output.txt b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/output.txt index 4e15f2629e2..9eff49bbd33 100644 --- a/acceptance/bundle/resources/grants/schemas/pipeline_run_as/output.txt +++ b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/output.txt @@ -1,5 +1,6 @@ -=== The grants of a referenced catalog or schema become an implicit lifecycle.depends_on +=== The grants of a referenced schema become an implicit lifecycle.depends_on + >>> [CLI] bundle validate -o json { "catalogs": { @@ -13,15 +14,12 @@ ] }, "schemas": { - "src": [ - { - "resource": "catalogs.pipeline_catalog.grants" - } - ] + "src": null } } === The plan orders dependents after the grants + >>> [CLI] bundle plan -o json { "resources.catalogs.pipeline_catalog": null, @@ -49,10 +47,6 @@ { "node": "resources.catalogs.pipeline_catalog", "label": "${resources.catalogs.pipeline_catalog.name}" - }, - { - "node": "resources.catalogs.pipeline_catalog.grants", - "label": "depends_on" } ], "resources.schemas.src.grants": [ @@ -73,9 +67,7 @@ Created schemas.src.grants Files: 3 uploaded, 0 deleted Resources: 5 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py --summary //unity-catalog/catalogs //unity-catalog/schemas //unity-catalog/permissions //pipelines -POST /api/2.1/unity-catalog/catalogs -PATCH /api/2.1/unity-catalog/permissions/catalog/catalog_grants_pipeline_[UNIQUE_NAME] +>>> print_requests.py --summary //unity-catalog/schemas //unity-catalog/permissions/schema //pipelines POST /api/2.1/unity-catalog/schemas PATCH /api/2.1/unity-catalog/permissions/schema/catalog_grants_pipeline_[UNIQUE_NAME].src POST /api/2.0/pipelines diff --git a/acceptance/bundle/resources/grants/schemas/pipeline_run_as/script b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/script index 392803e4258..975cdfd9ba0 100644 --- a/acceptance/bundle/resources/grants/schemas/pipeline_run_as/script +++ b/acceptance/bundle/resources/grants/schemas/pipeline_run_as/script @@ -1,7 +1,13 @@ # Creating the pipeline fails unless its run_as principal can already create tables in the # schema, so the pipeline must be deployed after the grants it relies on. # See https://github.com/databricks/cli/issues/4573 -export RUN_AS_SP="${TEST_SP_APPLICATION_ID:-6f1c2b3a-4d5e-4f60-8a7b-9c0d1e2f3a4b}" +if [ -z "$CLOUD_ENV" ]; then + export TEST_SP_APPLICATION_ID="22222222-2222-2222-2222-222222222222" +fi + +add_repl "$TEST_SP_APPLICATION_ID" TEST_SP_APPLICATION_ID + +export RUN_AS_SP="$TEST_SP_APPLICATION_ID" envsubst < databricks.yml.tmpl > databricks.yml @@ -10,13 +16,14 @@ cleanup() { } trap cleanup EXIT -title "The grants of a referenced catalog or schema become an implicit lifecycle.depends_on" +title "The grants of a referenced schema become an implicit lifecycle.depends_on" trace $CLI bundle validate -o json | jq '.resources | map_values(map_values(.lifecycle.depends_on))' title "The plan orders dependents after the grants" trace $CLI bundle plan -o json | jq '.plan | map_values(.depends_on)' trace $CLI bundle deploy -trace print_requests.py --summary //unity-catalog/catalogs //unity-catalog/schemas //unity-catalog/permissions //pipelines +# The catalog PATCH is excluded because its order relative to the schema is not fixed. +trace print_requests.py --summary //unity-catalog/schemas //unity-catalog/permissions/schema //pipelines trace $CLI bundle plan diff --git a/acceptance/bundle/resources/permissions/pipelines/depends_on/databricks.yml b/acceptance/bundle/resources/permissions/pipelines/depends_on/databricks.yml new file mode 100644 index 00000000000..2a17f078d0c --- /dev/null +++ b/acceptance/bundle/resources/permissions/pipelines/depends_on/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: pipeline-depends-on-permissions + +resources: + jobs: + notify: + name: notify + permissions: + - level: CAN_VIEW + group_name: users + + pipelines: + load: + # The pipeline refers to the job by name, so it waits for the job's permissions. + name: ${resources.jobs.notify.name}-load diff --git a/acceptance/bundle/resources/permissions/pipelines/depends_on/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/depends_on/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/permissions/pipelines/depends_on/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/pipelines/depends_on/output.txt b/acceptance/bundle/resources/permissions/pipelines/depends_on/output.txt new file mode 100644 index 00000000000..ff917c2fc80 --- /dev/null +++ b/acceptance/bundle/resources/permissions/pipelines/depends_on/output.txt @@ -0,0 +1,23 @@ + +=== A pipeline that refers to a job waits for the job's permissions + +>>> [CLI] bundle plan -o json +{ + "resources.jobs.notify": null, + "resources.jobs.notify.permissions": [ + { + "node": "resources.jobs.notify", + "label": "${resources.jobs.notify.id}" + } + ], + "resources.pipelines.load": [ + { + "node": "resources.jobs.notify", + "label": "${resources.jobs.notify.name}" + }, + { + "node": "resources.jobs.notify.permissions", + "label": "depends_on" + } + ] +} diff --git a/acceptance/bundle/resources/permissions/pipelines/depends_on/script b/acceptance/bundle/resources/permissions/pipelines/depends_on/script new file mode 100644 index 00000000000..4979de31b5d --- /dev/null +++ b/acceptance/bundle/resources/permissions/pipelines/depends_on/script @@ -0,0 +1,2 @@ +title "A pipeline that refers to a job waits for the job's permissions" +trace $CLI bundle plan -o json | jq '.plan | map_values(.depends_on)' From 8ecdfb855be88e185f03093cec1d87bcb202764d Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Sun, 15 Nov 2026 18:08:57 +0000 Subject: [PATCH 8/9] Drop the run_as privilege check from the pipeline fake The fake's create-time 403 only fires in some runs without the mutator, so the plan assertion in the pipeline_run_as acceptance test is the regression guard. Revert the fake to origin/main. Co-authored-by: Isaac --- libs/testserver/pipelines.go | 43 -------------- libs/testserver/pipelines_test.go | 94 ------------------------------- 2 files changed, 137 deletions(-) diff --git a/libs/testserver/pipelines.go b/libs/testserver/pipelines.go index 37b7668ad59..6960aebf350 100644 --- a/libs/testserver/pipelines.go +++ b/libs/testserver/pipelines.go @@ -1,13 +1,10 @@ package testserver import ( - "cmp" "encoding/json" "fmt" - "slices" "strings" - "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/pipelines" ) @@ -67,18 +64,6 @@ func (s *FakeWorkspace) PipelineCreate(req Request) Response { } r.Parameters = create.Parameters - // "schema" and "target" both name the target schema; only one of them can be set. - schemaName := spec.Catalog + "." + cmp.Or(spec.Schema, spec.Target) - if !s.runAsCanCreateTables(schemaName, create.RunAs) { - return Response{ - StatusCode: 403, - Body: map[string]string{ - "error_code": permissionDeniedErrorCode, - "message": fmt.Sprintf("User does not have CREATE TABLE on Schema '%s'.", schemaName), - }, - } - } - pipelineId := nextUUID() r.PipelineId = pipelineId r.CreatorUserName = "tester@databricks.com" @@ -103,34 +88,6 @@ func (s *FakeWorkspace) PipelineCreate(req Request) Response { } } -// runAsCanCreateTables reports whether the run_as principal of a new pipeline can create -// tables in its target schema, which the backend requires to create the pipeline. Only -// schemas created in this workspace are checked, because grants are tracked only for them. -func (s *FakeWorkspace) runAsCanCreateTables(schemaName string, runAs *pipelines.RunAs) bool { - var principal string - if runAs != nil { - principal = cmp.Or(runAs.ServicePrincipalName, runAs.UserName) - } - - schema, ok := s.Schemas[schemaName] - if principal == "" || !ok || principal == schema.Owner { - return true - } - - // Privileges granted on the catalog are inherited by its schemas. - for _, key := range []string{grantsKey("catalog", schema.CatalogName), grantsKey("schema", schemaName)} { - for _, assignment := range s.Grants[key] { - if assignment.Principal != principal { - continue - } - if slices.Contains(assignment.Privileges, catalog.PrivilegeCreateTable) || slices.Contains(assignment.Privileges, catalog.PrivilegeAllPrivileges) { - return true - } - } - } - return false -} - func setSpecDefaults(spec *pipelines.PipelineSpec, pipelineId string) { spec.Id = pipelineId // If the pipeline definition does not specify a catalog, it switches to Hive metastore mode diff --git a/libs/testserver/pipelines_test.go b/libs/testserver/pipelines_test.go index 12f0b2a07c6..6bb1dadfbdb 100644 --- a/libs/testserver/pipelines_test.go +++ b/libs/testserver/pipelines_test.go @@ -1,7 +1,6 @@ package testserver import ( - "strings" "testing" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -49,99 +48,6 @@ func TestPipelineCreate_AllowsSingleSegmentTargetSchemaName(t *testing.T) { assert.Equal(t, 0, response.StatusCode) } -func TestPipelineCreate_RequiresRunAsCreateTableOnSchema(t *testing.T) { - const pipeline = `{"name": "p", "catalog": "main", "schema": "src", "run_as": {"service_principal_name": "sp"}}` - - tests := []struct { - name string - createSchema bool - grants map[string]string - pipeline string - expectedStatusCode int - }{ - { - name: "no grants", - createSchema: true, - pipeline: pipeline, - expectedStatusCode: 403, - }, - { - name: "no grants with target", - createSchema: true, - pipeline: `{"name": "p", "catalog": "main", "target": "src", "run_as": {"service_principal_name": "sp"}}`, - expectedStatusCode: 403, - }, - { - name: "grants without CREATE_TABLE", - createSchema: true, - grants: map[string]string{"schema/main.src": "USE_SCHEMA"}, - pipeline: pipeline, - expectedStatusCode: 403, - }, - { - name: "CREATE_TABLE on schema", - createSchema: true, - grants: map[string]string{"schema/main.src": "CREATE_TABLE"}, - pipeline: pipeline, - }, - { - name: "ALL_PRIVILEGES on schema", - createSchema: true, - grants: map[string]string{"schema/main.src": "ALL_PRIVILEGES"}, - pipeline: pipeline, - }, - { - name: "CREATE_TABLE on catalog", - createSchema: true, - grants: map[string]string{"catalog/main": "CREATE_TABLE"}, - pipeline: pipeline, - }, - { - name: "run_as schema owner", - createSchema: true, - pipeline: `{"name": "p", "catalog": "main", "schema": "src", "run_as": {"user_name": "` + TestUser.UserName + `"}}`, - }, - { - name: "no run_as", - createSchema: true, - pipeline: `{"name": "p", "catalog": "main", "schema": "src"}`, - }, - { - name: "schema not created in workspace", - pipeline: pipeline, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // A user token, so that the schema owner is TestUser. - workspace := NewFakeWorkspace("http://test", UserNameTokenPrefix+"123") - - if tt.createSchema { - response := workspace.SchemasCreate(Request{Body: []byte(`{"catalog_name": "main", "name": "src"}`)}) - require.Equal(t, 0, response.StatusCode) - } - - for securable, privilege := range tt.grants { - securableType, fullName, _ := strings.Cut(securable, "/") - response := workspace.GrantsUpdate(Request{ - Body: []byte(`{"changes": [{"principal": "sp", "add": ["` + privilege + `"]}]}`), - }, securableType, fullName) - require.Equal(t, 0, response.StatusCode) - } - - response := workspace.PipelineCreate(Request{Body: []byte(tt.pipeline)}) - assert.Equal(t, tt.expectedStatusCode, response.StatusCode) - if tt.expectedStatusCode == 403 { - assert.Equal(t, map[string]string{ - "error_code": "PERMISSION_DENIED", - "message": "User does not have CREATE TABLE on Schema 'main.src'.", - }, response.Body) - } - }) - } -} - func TestPipelineStartUpdate_HandlesNonExistentPipeline(t *testing.T) { workspace := NewFakeWorkspace("http://test", "dbapi123") From 69173453fd346bf30e651dbda9dd4bf1d50ff5bb Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Sun, 15 Nov 2026 18:13:10 +0000 Subject: [PATCH 9/9] Format the pipeline dependency unit tests Co-authored-by: Isaac --- .../resourcemutator/capture_pipeline_dependencies_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bundle/config/mutator/resourcemutator/capture_pipeline_dependencies_test.go b/bundle/config/mutator/resourcemutator/capture_pipeline_dependencies_test.go index d51fd9925e6..382aeab9cd8 100644 --- a/bundle/config/mutator/resourcemutator/capture_pipeline_dependencies_test.go +++ b/bundle/config/mutator/resourcemutator/capture_pipeline_dependencies_test.go @@ -24,15 +24,15 @@ func TestCapturePipelineDependencies(t *testing.T) { // A pipeline that refers to the schema. A constructor, because the mutator writes to the // pipeline and a shared pointer would leak lifecycle.depends_on between cases. pipeline := func() *resources.Pipeline { - return &resources.Pipeline{CreatePipeline: pipelines.CreatePipeline{ + return &resources.Pipeline{ Catalog: "${resources.schemas.src.catalog_name}", Schema: "${resources.schemas.src.name}", - }} + } } // A job with permissions, referred to by name. job := func(permissions []resources.JobPermission) *resources.Job { return &resources.Job{ - JobSettings: jobs.JobSettings{Name: "job"}, + Name: "job", Permissions: permissions, } }