Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/editable-project-folder.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* A project folder after `-e` in job and pipeline environment dependencies, with optional extras such as `-e ./project[test]`, now maps to its deployed folder. Before, it failed with "unable to determine if ... is not a notebook". ([#7427](https://github.com/databricks/cli/pull/7427))
4 changes: 4 additions & 0 deletions acceptance/bundle/environments/dependencies/databricks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ resources:
- -r "${workspace.file_path}/Path With Spaces/requirements.txt"
- "-r '${workspace.file_path}/Path With Spaces/requirements.txt'"
- "-e ./file.py"
- "-e ./project"
- "-e ./project[test]"
- "-i http://myindexurl.com"
- "--index-url http://myindexurl.com"
- "test_package"
Expand All @@ -44,6 +46,8 @@ resources:
dependencies:
- "-r ./requirements.txt"
- "-e ./file.py"
- "-e ./project"
- "-e ./project[test]"
- "-i http://myindexurl.com"
- "--index-url http://myindexurl.com"
- "dist/*.whl"
Expand Down
10 changes: 9 additions & 1 deletion acceptance/bundle/environments/dependencies/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ Uploading dist/test.whl...
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files...
Created jobs.test_job
Created pipelines.test_pipeline
Files: 5 uploaded, 0 deleted
Files: 6 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments [OUTPUT_DIR]/out.requests.txt
Expand All @@ -27,6 +27,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
"-r \"/Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/Path With Spaces/requirements.txt\"",
"-r '/Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/Path With Spaces/requirements.txt'",
"-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/file.py",
"-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/project",
"-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/project[test]",
"-i http://myindexurl.com",
"--index-url http://myindexurl.com",
"test_package",
Expand Down Expand Up @@ -68,6 +70,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
"dependencies": [
"-r /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/requirements.txt",
"-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/file.py",
"-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/project",
"-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/project[test]",
"-i http://myindexurl.com",
"--index-url http://myindexurl.com",
"/Workspace/Users/[USERNAME]/.bundle/dependencies/default/artifacts/.internal/test.whl",
Expand Down Expand Up @@ -96,6 +100,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
"-r \"/Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/Path With Spaces/requirements.txt\"",
"-r '/Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/Path With Spaces/requirements.txt'",
"-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/file.py",
"-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/project",
"-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/project[test]",
"-i http://myindexurl.com",
"--index-url http://myindexurl.com",
"test_package",
Expand Down Expand Up @@ -127,6 +133,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
"dependencies": [
"-r /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/requirements.txt",
"-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/file.py",
"-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/project",
"-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/project[test]",
"-i http://myindexurl.com",
"--index-url http://myindexurl.com",
"./dist/*.whl",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
[project]
name = "orders"
version = "0.1.0"

[project.optional-dependencies]
test = ["faker"]
23 changes: 23 additions & 0 deletions bundle/config/mutator/translate_paths.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,15 @@ func (t *translateContext) rewritePath(
localPath = filepath.ToSlash(localPath)
localRelPath = filepath.ToSlash(localRelPath)

// The path after -e can also be a project folder, with optional extras such as ./project[test].
// The result is not cached: the cache key has no flag, so a later "-r ./project" would get the
// "-e" result instead of its error.
if opts.Mode == paths.TranslateModeEnvironmentPipFlag && flag == "-e" {
if interp, ok := t.translateEditableFolder(localRelPath); ok {
return interp, nil
}
}

// Convert local path into workspace path via specified function.
var interp string
switch opts.Mode {
Expand Down Expand Up @@ -251,6 +260,20 @@ func (t *translateContext) translateFilePath(ctx context.Context, literal, local
return path.Join(t.remoteRoot, localRelPath), nil
}

// translateEditableFolder maps a project folder after -e, with optional extras such as [test], to
// its deployed folder. It reports false if the path is not a folder in the sync root.
func (t *translateContext) translateEditableFolder(localRelPath string) (string, bool) {
folder, extras := localRelPath, ""
if i := strings.LastIndex(localRelPath, "["); i > 0 && strings.HasSuffix(localRelPath, "]") {
Comment thread
tharunbhuk-db marked this conversation as resolved.
Resolved
folder, extras = localRelPath[:i], localRelPath[i:]
}
info, err := t.b.SyncRoot.Stat(folder)
if err != nil || !info.IsDir() {
return "", false
}
return "-e " + path.Join(t.remoteRoot, folder) + extras, true
}

func (t *translateContext) translateDirectoryPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {
if t.skipLocalFileValidation {
return path.Join(t.remoteRoot, localRelPath), nil
Expand Down
48 changes: 48 additions & 0 deletions bundle/config/mutator/translate_paths_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -816,6 +816,54 @@ func TestTranslatePathJobEnvironments(t *testing.T) {
assert.Equal(t, "https://foo@bar.test/packages/pypi/simple", b.Config.Resources.Jobs["job"].Environments[0].Spec.Dependencies[6])
}

func TestTranslatePathEditableProjectFolder(t *testing.T) {
dir := t.TempDir()
touchEmptyFile(t, filepath.Join(dir, "project", "pyproject.toml"))
touchEmptyFile(t, filepath.Join(dir, "requirements.txt"))

translate := func(ctx context.Context, dependencies ...string) (*bundle.Bundle, diag.Diagnostics) {
b := &bundle.Bundle{
SyncRootPath: dir,
BundleRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle",
},
Resources: config.Resources{
Pipelines: map[string]*resources.Pipeline{
"pipeline": {
CreatePipeline: pipelines.CreatePipeline{
Environment: &pipelines.PipelinesEnvironment{Dependencies: dependencies},
},
},
},
},
},
}
bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "fake.yml")}})
return b, bundle.ApplySeq(ctx, b, mutator.NormalizePaths(), mutator.TranslatePaths())
}

ctx := t.Context()
b, diags := translate(ctx, "-e ./project", "-e ./project[test]", "-e .", "-e .[test]", "-r ./requirements.txt")
require.NoError(t, diags.Error())
assert.Equal(t, []string{
"-e /bundle/project",
"-e /bundle/project[test]",
"-e /bundle",
"-e /bundle[test]",
"-r /bundle/requirements.txt",
}, b.Config.Resources.Pipelines["pipeline"].Environment.Dependencies)

// -r takes a file, so a folder after it is still an error, also after the same folder after -e.
notebookErr := "unable to determine if " + filepath.Join(dir, "project") + " is not a notebook"
_, diags = translate(ctx, "-r ./project")
assert.ErrorContains(t, diags.Error(), notebookErr)
_, diags = translate(ctx, "-e ./project", "-r ./project")
assert.ErrorContains(t, diags.Error(), notebookErr)
}

func TestTranslatePathWithComplexVariables(t *testing.T) {
dir := t.TempDir()
b := &bundle.Bundle{
Expand Down
Loading