Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7286.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Fix `bundle validate` so workspace paths built from variables under `/Users` or `/Shared` receive the `/Workspace` prefix. ([#7286](https://github.com/databricks/cli/pull/7286))
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ variables:
workspace_root:
description: "root directory in the Databricks workspace to store the Declarative Automation Bundle and associated artifacts"
default: /Users/${workspace.current_user.userName}
config_file:
default: ${workspace.file_path}/path/to/config.yaml

targets:
dev:
Expand All @@ -23,5 +25,8 @@ resources:
tasks:
- existing_cluster_id: 500
python_wheel_task:
package_name: my_package
entry_point: main
named_parameters:
conf-file: "${workspace.file_path}/path/to/config.yaml"
alias-file: "${var.config_file}"
81 changes: 18 additions & 63 deletions acceptance/bundle/variables/prepend-workspace-var/output.txt
Original file line number Diff line number Diff line change
@@ -1,67 +1,22 @@
/Workspace should be prepended on all paths, but it is not the case:
Warning: required field "entry_point" is not set
at resources.jobs.my_job.tasks[0].python_wheel_task
in databricks.yml:26:13

Warning: required field "package_name" is not set
at resources.jobs.my_job.tasks[0].python_wheel_task
in databricks.yml:26:13
>>> [CLI] bundle validate -t dev -o json
{
"target": "dev",
"root_path": "/Workspace/Users/[USERNAME]/path/to/root",
"file_path": "/Workspace/Users/[USERNAME]/path/to/root/files",
"conf_file": "/Workspace/Users/[USERNAME]/path/to/root/files/path/to/config.yaml",
"alias_file": "/Workspace/Users/[USERNAME]/path/to/root/files/path/to/config.yaml"
}

>>> [CLI] bundle validate -t prod -o json
Warning: the bundle root path /Workspace/Shared/path/to/root is writable by all workspace users

The bundle is configured to use /Workspace/Shared, which will give read/write access to all users. If this is intentional, add CAN_MANAGE for 'group_name: users' permission to your bundle configuration. If the deployment should be restricted, move it to a restricted folder such as /Workspace/Users/<username or principal name>.

{
"bundle": {
"environment": "dev",
"git": {
"bundle_root_path": "."
},
"name": "test-bundle",
"target": "dev"
},
"resources": {
"jobs": {
"my_job": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Users/[USERNAME]/path/to/root/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"max_concurrent_runs": 1,
"name": "Untitled",
"queue": {
"enabled": true
},
"tasks": [
{
"existing_cluster_id": "500",
"python_wheel_task": {
"named_parameters": {
"conf-file": "/Users/[USERNAME]/path/to/root/files/path/to/config.yaml"
}
},
"task_key": ""
}
]
}
}
},
"sync": {
"paths": [
"."
]
},
"variables": {
"workspace_root": {
"default": "/Users/[USERNAME]",
"description": "root directory in the Databricks workspace to store the Declarative Automation Bundle and associated artifacts",
"value": "/Users/[USERNAME]"
}
},
"workspace": {
"artifact_path": "/Users/[USERNAME]/path/to/root/artifacts",
"file_path": "/Users/[USERNAME]/path/to/root/files",
"profile": "profile_name",
"resource_path": "/Users/[USERNAME]/path/to/root/resources",
"root_path": "/Users/[USERNAME]/path/to/root",
"state_path": "/Users/[USERNAME]/path/to/root/state"
}
"target": "prod",
"root_path": "/Workspace/Shared/path/to/root",
"file_path": "/Workspace/Shared/path/to/root/files",
"conf_file": "/Workspace/Shared/path/to/root/files/path/to/config.yaml",
"alias_file": "/Workspace/Shared/path/to/root/files/path/to/config.yaml"
}
5 changes: 3 additions & 2 deletions acceptance/bundle/variables/prepend-workspace-var/script
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
echo /Workspace should be prepended on all paths, but it is not the case: #2181
$CLI bundle validate -o json | jq 'del(.workspace.current_user)'
for target in dev prod; do
trace $CLI bundle validate -t "$target" -o json | jq '{target: .bundle.target, root_path: .workspace.root_path, file_path: .workspace.file_path, conf_file: .resources.jobs.my_job.tasks[0].python_wheel_task.named_parameters["conf-file"], alias_file: .resources.jobs.my_job.tasks[0].python_wheel_task.named_parameters["alias-file"]}'
done
4 changes: 4 additions & 0 deletions bundle/config/mutator/prepend_workspace_prefix.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ func prependWorkspacePrefixTo(path string) string {
return path
}

if path == "/Workspace" {
return path
}

for _, prefix := range skipPrefixes {
if strings.HasPrefix(path, prefix) {
return path
Expand Down
67 changes: 67 additions & 0 deletions bundle/config/mutator/prepend_workspace_prefix_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/variable"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/databricks-sdk-go/service/iam"
Expand All @@ -28,6 +29,10 @@ func TestPrependWorkspacePrefix(t *testing.T) {
path: "/Workspace/Users/test",
expected: "/Workspace/Users/test",
},
{
path: "/Workspace",
expected: "/Workspace",
},
{
path: "/Volumes/Users/test",
expected: "/Volumes/Users/test",
Expand Down Expand Up @@ -106,3 +111,65 @@ func TestPrependWorkspaceForDefaultConfig(t *testing.T) {
require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev/state", b.Config.Workspace.StatePath)
require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev/resources", b.Config.Workspace.ResourcePath)
}

func TestPrependWorkspacePrefixAfterVariableResolution(t *testing.T) {
for _, root := range []string{"/Users/jane@doe.com", "/Shared"} {
t.Run(root, func(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Bundle: config.Bundle{Name: root},
Workspace: config.Workspace{
RootPath: "${bundle.name}/path/to/root",
},
Variables: map[string]*variable.Variable{
"config_file": {Value: "${workspace.file_path}/config.yaml"},
},
},
}

diags := bundle.ApplySeq(t.Context(), b,
DefineDefaultWorkspacePaths(),
PrependWorkspacePrefix(),
ResolveVariableReferencesInWorkspacePaths(),
)
require.NoError(t, diags.Error())
require.Equal(t, root+"/path/to/root", b.Config.Workspace.RootPath)
require.Equal(t, "${workspace.file_path}/config.yaml", b.Config.Variables["config_file"].Value)

diags = bundle.ApplySeq(t.Context(), b, PrependWorkspacePrefix(), ResolveVariableReferencesWithoutResources())
require.NoError(t, diags.Error())
require.Equal(t, "/Workspace"+root+"/path/to/root", b.Config.Workspace.RootPath)
require.Equal(t, "/Workspace"+root+"/path/to/root/files", b.Config.Workspace.FilePath)
require.Equal(t, "/Workspace"+root+"/path/to/root/artifacts", b.Config.Workspace.ArtifactPath)
require.Equal(t, "/Workspace"+root+"/path/to/root/state", b.Config.Workspace.StatePath)
require.Equal(t, "/Workspace"+root+"/path/to/root/resources", b.Config.Workspace.ResourcePath)
require.Equal(t, "/Workspace"+root+"/path/to/root/files/config.yaml", b.Config.Variables["config_file"].Value)
})
}
}

func TestPrependWorkspacePrefixSourceLinkedAlias(t *testing.T) {
b := &bundle.Bundle{
SyncRootPath: "/Workspace/Users/jane/source",
Config: config.Root{
Workspace: config.Workspace{
RootPath: "/Users/jane/bundle",
FilePath: "${workspace.root_path}/files",
SyncMode: config.SyncModeInPlace,
},
Variables: map[string]*variable.Variable{
"config_file": {Value: "${workspace.file_path}/config.yaml"},
},
},
}

diags := bundle.ApplySeq(t.Context(), b,
PrependWorkspacePrefix(),
ResolveVariableReferencesInWorkspacePaths(),
PrependWorkspacePrefix(),
ResolveVariableReferencesWithoutResources(),
)
require.NoError(t, diags.Error())
require.Equal(t, "/Workspace/Users/jane/bundle/files", b.Config.Workspace.FilePath)
require.Equal(t, "/Workspace/Users/jane/source/config.yaml", b.Config.Variables["config_file"].Value)
}
30 changes: 23 additions & 7 deletions bundle/config/mutator/resolve_variable_references.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,8 @@ type resolveVariableReferences struct {

artifactsReferenceUsed bool

// excludePaths lists variable reference paths (e.g. "workspace.file_path") whose
// resolution should be skipped. References to these paths remain unresolved so a
// later mutator can set the value and re-run resolution.
excludePaths []string
// includePaths limits resolution to these configuration values when set.
includePaths []string
}

func ResolveVariableReferencesOnlyResources(prefixes ...string) bundle.Mutator {
Expand All @@ -91,6 +89,23 @@ func ResolveVariableReferencesWithoutResources(prefixes ...string) bundle.Mutato
}
}

// ResolveVariableReferencesInWorkspacePaths resolves paths before prefixing them.
func ResolveVariableReferencesInWorkspacePaths() bundle.Mutator {
return &resolveVariableReferences{
prefixes: defaultPrefixes,
lookupFn: lookup,
extraRounds: maxResolutionRounds - 1,
pattern: dyn.NewPattern(dyn.Key("workspace"), dyn.AnyKey()),
includePaths: []string{
"workspace.root_path",
"workspace.file_path",
"workspace.artifact_path",
"workspace.state_path",
"workspace.resource_path",
},
}
}

func ResolveVariableReferencesInLookup() bundle.Mutator {
return &resolveVariableReferences{
prefixes: defaultPrefixes,
Expand Down Expand Up @@ -222,6 +237,10 @@ func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []dyn

// If the pattern is nil, we resolve references in the entire configuration.
root, err := dyn.MapByPattern(root, m.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
if len(m.includePaths) > 0 && !slices.Contains(m.includePaths, p.String()) {
return v, nil
}

// Resolve variable references in all values.
return dynvar.Resolve(v, func(path dyn.Path) (dyn.Value, error) {
// Rewrite the shorthand path ${var.foo} into ${variables.foo.value}.
Expand All @@ -246,9 +265,6 @@ func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []dyn

// Perform resolution only if the path starts with one of the specified prefixes.
if slices.ContainsFunc(prefixes, path.HasPrefix) {
if slices.Contains(m.excludePaths, path.String()) {
return dyn.InvalidValue, dynvar.ErrSkipResolution
}
if m.allowPathFn != nil && !m.allowPathFn(path) {
return dyn.InvalidValue, dynvar.ErrSkipResolution
}
Expand Down
9 changes: 7 additions & 2 deletions bundle/phases/initialize.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,10 +125,15 @@ func Initialize(ctx context.Context, b *bundle.Bundle) {
// Updates (dynamic): variables.*.value (sets values based on resolved lookups)
mutator.ResolveLookupVariables(),

// Reads (dynamic): workspace paths and their variable references
// Updates (dynamic): workspace paths
// Resolve workspace paths before adding their prefixes.
mutator.ResolveVariableReferencesInWorkspacePaths(),
// Variable-based workspace paths can become absolute only after resolution.
mutator.PrependWorkspacePrefix(),
// Reads (dynamic): * (strings) (searches for variable references in string values)
// Updates (dynamic): * (except 'resources') (strings) (resolves variable references to their actual values)
// Resolves variable references in configuration (except resources) using bundle, workspace,
// and variables prefixes
// Resolve the remaining references after workspace path prefixes are final.
mutator.ResolveVariableReferencesWithoutResources(),

// Check for invalid use of /Volumes in workspace paths
Expand Down
Loading