diff --git a/.nextchanges/bundles/editable-project-folder.md b/.nextchanges/bundles/editable-project-folder.md new file mode 100644 index 00000000000..67c32a42019 --- /dev/null +++ b/.nextchanges/bundles/editable-project-folder.md @@ -0,0 +1 @@ +* A project folder after `-e` in job and pipeline environment dependencies, with optional extras such as `-e ./project[test]`, now maps to its deployed folder. Before, it failed with "unable to determine if ... is not a notebook". ([#7427](https://github.com/databricks/cli/pull/7427)) diff --git a/acceptance/bundle/environments/dependencies/databricks.yml b/acceptance/bundle/environments/dependencies/databricks.yml index 3fd067837f7..a76fca7bbf6 100644 --- a/acceptance/bundle/environments/dependencies/databricks.yml +++ b/acceptance/bundle/environments/dependencies/databricks.yml @@ -24,6 +24,8 @@ resources: - -r "${workspace.file_path}/Path With Spaces/requirements.txt" - "-r '${workspace.file_path}/Path With Spaces/requirements.txt'" - "-e ./file.py" + - "-e ./project" + - "-e ./project[test]" - "-i http://myindexurl.com" - "--index-url http://myindexurl.com" - "test_package" @@ -44,6 +46,8 @@ resources: dependencies: - "-r ./requirements.txt" - "-e ./file.py" + - "-e ./project" + - "-e ./project[test]" - "-i http://myindexurl.com" - "--index-url http://myindexurl.com" - "dist/*.whl" diff --git a/acceptance/bundle/environments/dependencies/output.txt b/acceptance/bundle/environments/dependencies/output.txt index 83a11d75cc8..5bab6f938ee 100644 --- a/acceptance/bundle/environments/dependencies/output.txt +++ b/acceptance/bundle/environments/dependencies/output.txt @@ -13,7 +13,7 @@ Uploading dist/test.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files... Created jobs.test_job Created pipelines.test_pipeline -Files: 5 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments [OUTPUT_DIR]/out.requests.txt @@ -27,6 +27,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "-r \"/Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/Path With Spaces/requirements.txt\"", "-r '/Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/Path With Spaces/requirements.txt'", "-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/file.py", + "-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/project", + "-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/project[test]", "-i http://myindexurl.com", "--index-url http://myindexurl.com", "test_package", @@ -68,6 +70,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "dependencies": [ "-r /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/requirements.txt", "-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/file.py", + "-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/project", + "-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/project[test]", "-i http://myindexurl.com", "--index-url http://myindexurl.com", "/Workspace/Users/[USERNAME]/.bundle/dependencies/default/artifacts/.internal/test.whl", @@ -96,6 +100,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "-r \"/Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/Path With Spaces/requirements.txt\"", "-r '/Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/Path With Spaces/requirements.txt'", "-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/file.py", + "-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/project", + "-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/project[test]", "-i http://myindexurl.com", "--index-url http://myindexurl.com", "test_package", @@ -127,6 +133,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "dependencies": [ "-r /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/requirements.txt", "-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/file.py", + "-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/project", + "-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/project[test]", "-i http://myindexurl.com", "--index-url http://myindexurl.com", "./dist/*.whl", diff --git a/acceptance/bundle/environments/dependencies/project/pyproject.toml b/acceptance/bundle/environments/dependencies/project/pyproject.toml new file mode 100644 index 00000000000..468fddd1ca7 --- /dev/null +++ b/acceptance/bundle/environments/dependencies/project/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "orders" +version = "0.1.0" + +[project.optional-dependencies] +test = ["faker"] diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 956d010ec95..5ad9510692c 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -162,6 +162,15 @@ func (t *translateContext) rewritePath( localPath = filepath.ToSlash(localPath) localRelPath = filepath.ToSlash(localRelPath) + // The path after -e can also be a project folder, with optional extras such as ./project[test]. + // The result is not cached: the cache key has no flag, so a later "-r ./project" would get the + // "-e" result instead of its error. + if opts.Mode == paths.TranslateModeEnvironmentPipFlag && flag == "-e" { + if interp, ok := t.translateEditableFolder(localRelPath); ok { + return interp, nil + } + } + // Convert local path into workspace path via specified function. var interp string switch opts.Mode { @@ -251,6 +260,20 @@ func (t *translateContext) translateFilePath(ctx context.Context, literal, local return path.Join(t.remoteRoot, localRelPath), nil } +// translateEditableFolder maps a project folder after -e, with optional extras such as [test], to +// its deployed folder. It reports false if the path is not a folder in the sync root. +func (t *translateContext) translateEditableFolder(localRelPath string) (string, bool) { + folder, extras := localRelPath, "" + if i := strings.LastIndex(localRelPath, "["); i > 0 && strings.HasSuffix(localRelPath, "]") { + folder, extras = localRelPath[:i], localRelPath[i:] + } + info, err := t.b.SyncRoot.Stat(folder) + if err != nil || !info.IsDir() { + return "", false + } + return "-e " + path.Join(t.remoteRoot, folder) + extras, true +} + func (t *translateContext) translateDirectoryPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) { if t.skipLocalFileValidation { return path.Join(t.remoteRoot, localRelPath), nil diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 8aaf9ce9c71..106662ed802 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -816,6 +816,54 @@ func TestTranslatePathJobEnvironments(t *testing.T) { assert.Equal(t, "https://foo@bar.test/packages/pypi/simple", b.Config.Resources.Jobs["job"].Environments[0].Spec.Dependencies[6]) } +func TestTranslatePathEditableProjectFolder(t *testing.T) { + dir := t.TempDir() + touchEmptyFile(t, filepath.Join(dir, "project", "pyproject.toml")) + touchEmptyFile(t, filepath.Join(dir, "requirements.txt")) + + translate := func(ctx context.Context, dependencies ...string) (*bundle.Bundle, diag.Diagnostics) { + b := &bundle.Bundle{ + SyncRootPath: dir, + BundleRootPath: dir, + SyncRoot: vfs.MustNew(dir), + Config: config.Root{ + Workspace: config.Workspace{ + FilePath: "/bundle", + }, + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "pipeline": { + CreatePipeline: pipelines.CreatePipeline{ + Environment: &pipelines.PipelinesEnvironment{Dependencies: dependencies}, + }, + }, + }, + }, + }, + } + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "fake.yml")}}) + return b, bundle.ApplySeq(ctx, b, mutator.NormalizePaths(), mutator.TranslatePaths()) + } + + ctx := t.Context() + b, diags := translate(ctx, "-e ./project", "-e ./project[test]", "-e .", "-e .[test]", "-r ./requirements.txt") + require.NoError(t, diags.Error()) + assert.Equal(t, []string{ + "-e /bundle/project", + "-e /bundle/project[test]", + "-e /bundle", + "-e /bundle[test]", + "-r /bundle/requirements.txt", + }, b.Config.Resources.Pipelines["pipeline"].Environment.Dependencies) + + // -r takes a file, so a folder after it is still an error, also after the same folder after -e. + notebookErr := "unable to determine if " + filepath.Join(dir, "project") + " is not a notebook" + _, diags = translate(ctx, "-r ./project") + assert.ErrorContains(t, diags.Error(), notebookErr) + _, diags = translate(ctx, "-e ./project", "-r ./project") + assert.ErrorContains(t, diags.Error(), notebookErr) +} + func TestTranslatePathWithComplexVariables(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{