From 1735f2da94aeed92155d7a84da2b1f1430606cf3 Mon Sep 17 00:00:00 2001 From: "chenyue.ma" Date: Wed, 28 Oct 2026 22:30:25 +0000 Subject: [PATCH 1/9] Support file selectors in pipelines run --- .nextchanges/cli/pipeline-refresh-by-file.md | 1 + .../pipelines/run/refresh-flags/output.txt | 102 ++++++++++- acceptance/pipelines/run/refresh-flags/script | 19 ++ bundle/run/pipeline_options.go | 25 ++- bundle/run/pipeline_options_test.go | 77 ++++++++ cmd/pipelines/refresh.go | 105 +++++++++++ cmd/pipelines/refresh_test.go | 170 ++++++++++++++++++ cmd/pipelines/run.go | 26 ++- cmd/pipelines/run_test.go | 28 +++ cmd/pipelines/templates.go | 8 +- libs/testserver/fake_workspace.go | 4 +- libs/testserver/handlers.go | 2 +- libs/testserver/pipelines.go | 28 ++- libs/testserver/pipelines_test.go | 44 ++++- 14 files changed, 612 insertions(+), 27 deletions(-) create mode 100644 .nextchanges/cli/pipeline-refresh-by-file.md create mode 100644 cmd/pipelines/refresh.go create mode 100644 cmd/pipelines/refresh_test.go diff --git a/.nextchanges/cli/pipeline-refresh-by-file.md b/.nextchanges/cli/pipeline-refresh-by-file.md new file mode 100644 index 00000000000..c5e25a11058 --- /dev/null +++ b/.nextchanges/cli/pipeline-refresh-by-file.md @@ -0,0 +1 @@ +* Support source files in `pipelines run --refresh` and `--full-refresh` selectors. ([#7184](https://github.com/databricks/cli/pull/7184)) diff --git a/acceptance/pipelines/run/refresh-flags/output.txt b/acceptance/pipelines/run/refresh-flags/output.txt index 791a73fa999..b821445daa3 100644 --- a/acceptance/pipelines/run/refresh-flags/output.txt +++ b/acceptance/pipelines/run/refresh-flags/output.txt @@ -20,7 +20,7 @@ Update ID: [UUID][1] Update for pipeline completed successfully. Pipeline configurations for this update: -• All tables are refreshed +• Refreshed [table1, table2] >>> print_requests.py //pipelines { @@ -48,7 +48,7 @@ Update ID: [UUID][2] Update for pipeline completed successfully. Pipeline configurations for this update: -• All tables are refreshed +• All tables are fully refreshed >>> print_requests.py //pipelines { @@ -73,7 +73,7 @@ Update ID: [UUID][3] Update for pipeline completed successfully. Pipeline configurations for this update: -• All tables are refreshed +• Full refreshed [table1, table2] >>> print_requests.py //pipelines { @@ -101,7 +101,8 @@ Update ID: [UUID][4] Update for pipeline completed successfully. Pipeline configurations for this update: -• All tables are refreshed +• Refreshed [table3, table4] +• Full refreshed [table1, table2] >>> print_requests.py //pipelines { @@ -118,3 +119,96 @@ Pipeline configurations for this update: ] } } + +=== Refreshing tables defined in a source file +>>> [CLI] pipelines run --refresh pipeline_file.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] + +Update ID: [UUID] + +Update for pipeline completed successfully. + +Pipeline configurations for this update: +• Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files/pipeline_file.py] + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files/pipeline_file.py" + ] + } +} + +=== Fully refreshing tables defined in a source file +>>> [CLI] pipelines run --full-refresh pipeline_file.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] + +Update ID: [UUID] + +Update for pipeline completed successfully. + +Pipeline configurations for this update: +• Full refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files/pipeline_file.py] + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "full_refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files/pipeline_file.py" + ] + } +} + +=== Rejecting missing sources and incompatible refresh modes +>>> musterr [CLI] pipelines run --refresh ./missing.sql --no-wait +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: invalid --refresh: source file "./missing.sql" not found; use a table name or an existing source file + +>>> musterr [CLI] pipelines run --refresh pipeline_file.py --full-refresh-all --no-wait +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: cannot use --refresh or --full-refresh together with --full-refresh-all, these flags are mutually exclusive + +>>> musterr [CLI] pipelines run --full-refresh ./missing.sql --restart --no-wait +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: invalid --full-refresh: source file "./missing.sql" not found; use a table name or an existing source file + +>>> test ! -e [OUTPUT_DIR]/out.requests.txt + +=== Rejecting mixed selectors before restarting the pipeline +>>> musterr [CLI] pipelines run --refresh table1,pipeline_file.py --restart --no-wait +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: cannot mix table names and source files in --refresh; use only table names or only source files + +>>> musterr [CLI] pipelines run --full-refresh table2,pipeline_file.py --restart --no-wait +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: cannot mix table names and source files in --full-refresh; use only table names or only source files + +>>> test ! -e [OUTPUT_DIR]/out.requests.txt diff --git a/acceptance/pipelines/run/refresh-flags/script b/acceptance/pipelines/run/refresh-flags/script index b324074d59f..e2838af2063 100644 --- a/acceptance/pipelines/run/refresh-flags/script +++ b/acceptance/pipelines/run/refresh-flags/script @@ -16,3 +16,22 @@ trace print_requests.py //pipelines title "Running pipeline with --full-refresh flag and --refresh flag" trace $CLI pipelines run --full-refresh table1,table2 --refresh table3,table4 trace print_requests.py //pipelines + +title "Refreshing tables defined in a source file" +trace $CLI pipelines run --refresh pipeline_file.py +trace print_requests.py //pipelines + +title "Fully refreshing tables defined in a source file" +trace $CLI pipelines run --full-refresh pipeline_file.py +trace print_requests.py //pipelines + +title "Rejecting missing sources and incompatible refresh modes" +trace musterr $CLI pipelines run --refresh ./missing.sql --no-wait +trace musterr $CLI pipelines run --refresh pipeline_file.py --full-refresh-all --no-wait +trace musterr $CLI pipelines run --full-refresh ./missing.sql --restart --no-wait +trace test ! -e "$OUT_REQUESTS" + +title "Rejecting mixed selectors before restarting the pipeline" +trace musterr $CLI pipelines run --refresh table1,pipeline_file.py --restart --no-wait +trace musterr $CLI pipelines run --full-refresh table2,pipeline_file.py --restart --no-wait +trace test ! -e "$OUT_REQUESTS" diff --git a/bundle/run/pipeline_options.go b/bundle/run/pipeline_options.go index 63ff3e81289..164e8a2de61 100644 --- a/bundle/run/pipeline_options.go +++ b/bundle/run/pipeline_options.go @@ -1,6 +1,7 @@ package run import ( + "errors" "fmt" "strings" @@ -17,12 +18,16 @@ type PipelineOptions struct { // List of tables to update. Refresh []string + RefreshByPath []string + // Perform a full graph reset and recompute. FullRefreshAll bool // List of tables to reset and recompute. FullRefresh []string + FullRefreshByPath []string + // Perform an update to validate graph correctness. ValidateOnly bool } @@ -51,10 +56,18 @@ func (o *PipelineOptions) Validate(pipeline *resources.Pipeline) error { if len(mutuallyExclusiveFlags) > 1 { return fmt.Errorf("%s pipeline run flags are mutually exclusive", strings.Join(mutuallyExclusiveFlags, ", ")) - } else if len(mutuallyExclusiveFlags) == 1 && (len(o.Refresh) > 0 || len(o.FullRefresh) > 0) { + } else if len(mutuallyExclusiveFlags) == 1 && (len(o.Refresh) > 0 || len(o.FullRefresh) > 0 || len(o.RefreshByPath) > 0 || len(o.FullRefreshByPath) > 0) { return fmt.Errorf("cannot use --refresh or --full-refresh together with %s, these flags are mutually exclusive", strings.Join(mutuallyExclusiveFlags, "")) } + // The API ignores file selections when table names are supplied for the same refresh mode. + if len(o.Refresh) > 0 && len(o.RefreshByPath) > 0 { + return errors.New("cannot mix table names and source files in --refresh; use only table names or only source files") + } + if len(o.FullRefresh) > 0 && len(o.FullRefreshByPath) > 0 { + return errors.New("cannot mix table names and source files in --full-refresh; use only table names or only source files") + } + return nil } @@ -66,10 +79,12 @@ func (o *PipelineOptions) toPayload(pipeline *resources.Pipeline, pipelineID str PipelineId: pipelineID, // Note: `RefreshAll` is implied if the fields below are not set. - RefreshSelection: o.Refresh, - FullRefresh: o.FullRefreshAll, - FullRefreshSelection: o.FullRefresh, - ValidateOnly: o.ValidateOnly, + RefreshSelection: o.Refresh, + FullRefresh: o.FullRefreshAll, + FullRefreshSelection: o.FullRefresh, + RefreshSelectionByPath: o.RefreshByPath, + FullRefreshSelectionByPath: o.FullRefreshByPath, + ValidateOnly: o.ValidateOnly, // Only true is sent. When false the field is omitted, so the backend keeps // using the pipeline-level development property while it is being deprecated. diff --git a/bundle/run/pipeline_options_test.go b/bundle/run/pipeline_options_test.go index 1eda4c8bae7..b2f7c0a3f81 100644 --- a/bundle/run/pipeline_options_test.go +++ b/bundle/run/pipeline_options_test.go @@ -1,6 +1,7 @@ package run import ( + "encoding/json" "testing" flag "github.com/spf13/pflag" @@ -8,6 +9,77 @@ import ( "github.com/stretchr/testify/require" ) +func TestPipelineOptionsPathSelectionsPayload(t *testing.T) { + tests := []struct { + name string + opts PipelineOptions + expected string + }{ + { + name: "file selections for both refresh modes", + opts: PipelineOptions{ + RefreshByPath: []string{"/Workspace/Shared/source.sql"}, + FullRefreshByPath: []string{"/Workspace/Shared/notebook"}, + }, + expected: `{"refresh_selection_by_path":["/Workspace/Shared/source.sql"],"full_refresh_selection_by_path":["/Workspace/Shared/notebook"]}`, + }, + { + name: "table refresh and file full-refresh", + opts: PipelineOptions{ + Refresh: []string{"catalog.schema.table1"}, + FullRefreshByPath: []string{"/Workspace/Shared/notebook"}, + }, + expected: `{"refresh_selection":["catalog.schema.table1"],"full_refresh_selection_by_path":["/Workspace/Shared/notebook"]}`, + }, + { + name: "file refresh and table full-refresh", + opts: PipelineOptions{ + RefreshByPath: []string{"/Workspace/Shared/source.sql"}, + FullRefresh: []string{"catalog.schema.table2"}, + }, + expected: `{"refresh_selection_by_path":["/Workspace/Shared/source.sql"],"full_refresh_selection":["catalog.schema.table2"]}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + payload, err := tt.opts.toPayload(nil, "pipeline-id", false) + require.NoError(t, err) + data, err := json.Marshal(payload) + require.NoError(t, err) + assert.JSONEq(t, tt.expected, string(data)) + }) + } +} + +func TestPipelineOptionsRejectMixedSelectors(t *testing.T) { + tests := []struct { + name string + opts PipelineOptions + }{ + { + name: "--refresh", + opts: PipelineOptions{ + Refresh: []string{"table1"}, + RefreshByPath: []string{"/Workspace/Shared/source.sql"}, + }, + }, + { + name: "--full-refresh", + opts: PipelineOptions{ + FullRefresh: []string{"table1"}, + FullRefreshByPath: []string{"/Workspace/Shared/source.sql"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + payload, err := tt.opts.toPayload(nil, "pipeline-id", false) + require.ErrorContains(t, err, "cannot mix table names and source files in "+tt.name) + assert.Nil(t, payload) + }) + } +} + func setupPipelineOptions(t *testing.T) (*flag.FlagSet, *PipelineOptions) { var fs flag.FlagSet var opts PipelineOptions @@ -115,6 +187,11 @@ func TestPipelineOptionsValidateFailureWithRefreshAndMutuallyExclusiveFlags(t *t require.NoError(t, err) err = opts.Validate(nil) assert.ErrorContains(t, err, "cannot use --refresh or --full-refresh together with") + + opts.RefreshByPath, opts.Refresh = opts.Refresh, nil + opts.FullRefreshByPath, opts.FullRefresh = opts.FullRefresh, nil + err = opts.Validate(nil) + assert.ErrorContains(t, err, "cannot use --refresh or --full-refresh together with") } } } diff --git a/cmd/pipelines/refresh.go b/cmd/pipelines/refresh.go new file mode 100644 index 00000000000..d26da6e33a8 --- /dev/null +++ b/cmd/pipelines/refresh.go @@ -0,0 +1,105 @@ +package pipelines + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/notebook" + "github.com/databricks/cli/libs/sync" +) + +// resolveRefreshSelectors separates table names from source files and translates local files to workspace paths. +func resolveRefreshSelectors(ctx context.Context, b *bundle.Bundle, selectors []string) (tables, paths []string, err error) { + var remoteNames map[string]string + for _, selector := range selectors { + if strings.HasPrefix(selector, "/Workspace/") { + paths = append(paths, selector) + continue + } + + // Quoted identifiers remain table selectors even when they contain path separators. + if strings.Contains(selector, "`") { + tables = append(tables, selector) + continue + } + + info, statErr := os.Stat(selector) + // A bare selector is a table unless it identifies an existing local file. + looksLikeFile := strings.ContainsAny(selector, `/\`) + if !looksLikeFile && (statErr != nil || info.IsDir()) { + tables = append(tables, selector) + continue + } + if errors.Is(statErr, fs.ErrNotExist) { + return nil, nil, fmt.Errorf("source file %q not found; use a table name or an existing source file", selector) + } + if statErr != nil { + return nil, nil, fmt.Errorf("cannot read refresh selector %q: %w", selector, statErr) + } + if info.IsDir() { + return nil, nil, fmt.Errorf("refresh selector %q is a directory; select a source file", selector) + } + + localPath, err := filepath.Abs(selector) + if err != nil { + return nil, nil, err + } + rel, err := filepath.Rel(b.SyncRootPath, localPath) + if err != nil { + return nil, nil, err + } + if !filepath.IsLocal(rel) { + return nil, nil, fmt.Errorf("source file %q is outside the bundle sync root", selector) + } + if remoteNames == nil { + remoteNames, err = refreshSourcePaths(ctx, b) + if err != nil { + return nil, nil, err + } + } + remoteName, ok := remoteNames[filepath.ToSlash(rel)] + if !ok { + if _, _, err := notebook.DetectWithFS(b.SyncRoot, filepath.ToSlash(rel)); err != nil { + return nil, nil, fmt.Errorf("cannot identify source file %q: %w", selector, err) + } + return nil, nil, fmt.Errorf("source file %q is excluded from bundle sync; include it and run 'databricks pipelines deploy' first", selector) + } + remoteRoot := b.Config.Workspace.FilePath + if b.IsSourceLinkedDeployment() { + remoteRoot = filepath.ToSlash(b.SyncRootPath) + } + if strings.Contains(remoteRoot, "${") { + return nil, nil, fmt.Errorf("cannot resolve the deployed path for %q; use its absolute /Workspace/ path", selector) + } + paths = append(paths, path.Join(remoteRoot, remoteName)) + } + return tables, paths, nil +} + +// refreshSourcePaths uses the sync mapping so notebook names and sync exclusions match deployment. +func refreshSourcePaths(ctx context.Context, b *bundle.Bundle) (map[string]string, error) { + includes, err := b.GetSyncIncludePatterns(ctx) + if err != nil { + return nil, err + } + list, err := sync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, includes, b.Config.Sync.Exclude) + if err != nil { + return nil, err + } + files, err := list.Files(ctx) + if err != nil { + return nil, err + } + state, err := sync.NewSnapshotState(files) + if err != nil { + return nil, err + } + return state.LocalToRemoteNames, nil +} diff --git a/cmd/pipelines/refresh_test.go b/cmd/pipelines/refresh_test.go new file mode 100644 index 00000000000..eb1824a5890 --- /dev/null +++ b/cmd/pipelines/refresh_test.go @@ -0,0 +1,170 @@ +package pipelines + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/vfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveRefreshSelectors(t *testing.T) { + longTable := strings.Repeat("c", 100) + "." + strings.Repeat("s", 100) + "." + strings.Repeat("t", 100) + tests := []struct { + name string + selectors []string + tables []string + paths []string + err string + }{ + {name: "no selection"}, + {name: "long qualified table", selectors: []string{longTable}, tables: []string{longTable}}, + { + name: "table identifiers", + selectors: []string{"table1", "schema.table2", "schema.sql", "catalog.schema.sql", "`schema`.`table/with/slashes`", "`schema`.`sql`"}, + tables: []string{"table1", "schema.table2", "schema.sql", "catalog.schema.sql", "`schema`.`table/with/slashes`", "`schema`.`sql`"}, + }, + { + name: "quoted table with the same name as a file", + selectors: []string{"`src/query.sql`"}, + tables: []string{"`src/query.sql`"}, + }, + { + name: "mixed table and file selectors", + selectors: []string{"table1", "src/query.sql", "src/module.py"}, + tables: []string{"table1"}, + paths: []string{"/Workspace/Shared/project/files/src/query.sql", "/Workspace/Shared/project/files/src/module.py"}, + }, + { + name: "notebook extension removed", + selectors: []string{"src/notebook.py", "src/jupyter.ipynb"}, + paths: []string{"/Workspace/Shared/project/files/src/notebook", "/Workspace/Shared/project/files/src/jupyter"}, + }, + { + name: "workspace path", + selectors: []string{"/Workspace/Shared/remote.sql"}, + paths: []string{"/Workspace/Shared/remote.sql"}, + }, + {name: "missing relative file", selectors: []string{"src/missing.sql"}, err: `source file "src/missing.sql" not found`}, + {name: "explicit missing filename", selectors: []string{"./missing.sql"}, err: `source file "./missing.sql" not found`}, + {name: "table named like a directory", selectors: []string{"src"}, tables: []string{"src"}}, + {name: "explicit directory", selectors: []string{"./src"}, err: "is a directory"}, + {name: "excluded file", selectors: []string{"src/excluded.sql"}, err: "is excluded from bundle sync"}, + {name: "invalid notebook", selectors: []string{"src/invalid.ipynb"}, err: "invalid Jupyter notebook file"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "src"), 0o700)) + for name, content := range map[string]string{ + "query.sql": "SELECT 1;\n", + "module.py": "print('file')\n", + "notebook.py": "# Databricks notebook source\nprint('notebook')\n", + "jupyter.ipynb": `{"cells":[],"metadata":{},"nbformat":4,"nbformat_minor":5}`, + "excluded.sql": "SELECT 2;\n", + "invalid.ipynb": `{}`, + } { + require.NoError(t, os.WriteFile(filepath.Join(root, "src", name), []byte(content), 0o600)) + } + t.Chdir(root) + b := refreshTestBundle(root) + b.Config.Sync.Exclude = []string{"src/excluded.sql"} + tables, paths, err := resolveRefreshSelectors(t.Context(), b, tt.selectors) + if tt.err != "" { + require.ErrorContains(t, err, tt.err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.tables, tables) + assert.Equal(t, tt.paths, paths) + }) + } +} + +func refreshTestBundle(root string) *bundle.Bundle { + return &bundle.Bundle{ + BundleRootPath: root, + SyncRootPath: root, + SyncRoot: vfs.MustNew(root), + WorktreeRoot: vfs.MustNew(root), + Config: config.Root{ + Bundle: config.Bundle{Name: "test", Target: "default"}, + Workspace: config.Workspace{FilePath: "/Workspace/Shared/project/files"}, + Sync: config.Sync{Paths: []string{"."}}, + }, + } +} + +func TestResolveRefreshSelectorsFromSubdirectory(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "project", "src"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(root, "shared.sql"), []byte("SELECT 1;\n"), 0o600)) + t.Chdir(filepath.Join(root, "project", "src")) + b := refreshTestBundle(root) + b.BundleRootPath = filepath.Join(root, "project") + for _, selector := range []string{"../../shared.sql", filepath.Join(root, "shared.sql")} { + tables, paths, err := resolveRefreshSelectors(t.Context(), b, []string{selector}) + require.NoError(t, err) + assert.Empty(t, tables) + assert.Equal(t, []string{"/Workspace/Shared/project/files/shared.sql"}, paths) + } +} + +func TestResolveRefreshSelectorsOutsideSyncRoot(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(root, "project"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(root, "outside.sql"), []byte("SELECT 1;\n"), 0o600)) + t.Chdir(filepath.Join(root, "project")) + _, _, err := resolveRefreshSelectors(t.Context(), refreshTestBundle(filepath.Join(root, "project")), []string{"../outside.sql"}) + require.ErrorContains(t, err, "is outside the bundle sync root") +} + +func TestResolveRefreshSelectorsDeploymentMode(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "query.sql"), []byte("SELECT 1;\n"), 0o600)) + t.Chdir(root) + enabled, disabled := true, false + localPath := filepath.ToSlash(filepath.Join(root, "query.sql")) + remotePath := "/Workspace/Shared/project/files/query.sql" + tests := []struct { + name string + mode config.SyncMode + sourceLinked *bool + want string + }{ + {name: "default upload", want: remotePath}, + {name: "in-place", mode: config.SyncModeInPlace, want: localPath}, + {name: "legacy source-linked", sourceLinked: &enabled, want: localPath}, + {name: "overwrite overrides legacy setting", mode: config.SyncModeOverwrite, sourceLinked: &enabled, want: remotePath}, + {name: "in-place overrides legacy setting", mode: config.SyncModeInPlace, sourceLinked: &disabled, want: localPath}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := refreshTestBundle(root) + b.Config.Workspace.SyncMode = tt.mode + b.Config.Presets.SourceLinkedDeployment = tt.sourceLinked + tables, paths, err := resolveRefreshSelectors(t.Context(), b, []string{"query.sql"}) + require.NoError(t, err) + assert.Empty(t, tables) + assert.Equal(t, []string{tt.want}, paths) + }) + } +} + +func TestResolveRefreshSelectorsUnresolvedWorkspacePath(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "query.sql"), []byte("SELECT 1;\n"), 0o600)) + t.Chdir(root) + b := refreshTestBundle(root) + b.Config.Workspace.FilePath = "${resources.internal_immutable_snapshots.immutable.full_path}/files" + b.Config.Workspace.SyncMode = config.SyncModeImmutable + enabled := true + b.Config.Presets.SourceLinkedDeployment = &enabled + _, _, err := resolveRefreshSelectors(t.Context(), b, []string{"query.sql"}) + require.ErrorContains(t, err, "use its absolute /Workspace/ path") +} diff --git a/cmd/pipelines/run.go b/cmd/pipelines/run.go index 9f81961b6f0..702a8392edf 100644 --- a/cmd/pipelines/run.go +++ b/cmd/pipelines/run.go @@ -233,7 +233,14 @@ func runCommand() *cobra.Command { Long: `Run the pipeline identified by KEY. KEY is the unique name of the pipeline to run, as defined in its YAML file. If there is only one pipeline in the project, KEY is optional and the pipeline will be auto-selected. -Refreshes all tables in the pipeline unless otherwise specified.`, +Refreshes all tables in the pipeline unless otherwise specified. + +Use --refresh or --full-refresh with table names, local source files, or absolute +/Workspace/ paths. Local files use the bundle sync configuration and paths are +relative to the current directory. Deploy local changes before running. +Each flag accepts either table names or source files, but cannot mix both. +Existing local files take precedence over unquoted table names. Use ./ to require +a file, or quote table identifiers with backticks to select an ambiguous table.`, } var refresh []string @@ -241,9 +248,9 @@ Refreshes all tables in the pipeline unless otherwise specified.`, var fullRefresh []string pipelineGroup := cmdgroup.NewFlagGroup("Pipeline Run") - pipelineGroup.FlagSet().StringSliceVar(&refresh, "refresh", nil, "List of tables to run.") + pipelineGroup.FlagSet().StringSliceVar(&refresh, "refresh", nil, "List of tables or source files to run. Local paths are relative to the current directory.") pipelineGroup.FlagSet().BoolVar(&fullRefreshAll, "full-refresh-all", false, "Perform a full graph reset and recompute.") - pipelineGroup.FlagSet().StringSliceVar(&fullRefresh, "full-refresh", nil, "List of tables to reset and recompute.") + pipelineGroup.FlagSet().StringSliceVar(&fullRefresh, "full-refresh", nil, "List of tables or source files to reset and recompute. Local paths are relative to the current directory.") wrappedCmd := cmdgroup.NewCommandWithGroupFlag(cmd) wrappedCmd.AddFlagGroup(pipelineGroup) @@ -280,12 +287,21 @@ Refreshes all tables in the pipeline unless otherwise specified.`, runOptions := run.Options{ Pipeline: run.PipelineOptions{ - Refresh: refresh, FullRefreshAll: fullRefreshAll, - FullRefresh: fullRefresh, }, NoWait: noWait, } + runOptions.Pipeline.Refresh, runOptions.Pipeline.RefreshByPath, err = resolveRefreshSelectors(ctx, b, refresh) + if err != nil { + return fmt.Errorf("invalid --refresh: %w", err) + } + runOptions.Pipeline.FullRefresh, runOptions.Pipeline.FullRefreshByPath, err = resolveRefreshSelectors(ctx, b, fullRefresh) + if err != nil { + return fmt.Errorf("invalid --full-refresh: %w", err) + } + if err := runOptions.Pipeline.Validate(nil); err != nil { + return err + } var runOutput bundlerunoutput.RunOutput if restart { diff --git a/cmd/pipelines/run_test.go b/cmd/pipelines/run_test.go index 9937cb83d0d..2c910669e34 100644 --- a/cmd/pipelines/run_test.go +++ b/cmd/pipelines/run_test.go @@ -99,6 +99,34 @@ Update for pipeline completed successfully. Pipeline configurations for this update: • Refreshed [table1, table2] • Full refreshed [table3] +`, + }, + { + name: "pipeline update with file refresh selections", + update: pipelines.UpdateInfo{ + RefreshSelectionByPath: []string{"/Workspace/Shared/source.sql"}, + FullRefreshSelectionByPath: []string{"/Workspace/Shared/notebook"}, + }, + expected: ` +Update for pipeline completed successfully. + +Pipeline configurations for this update: +• Refreshed tables defined in [/Workspace/Shared/source.sql] +• Full refreshed tables defined in [/Workspace/Shared/notebook] +`, + }, + { + name: "pipeline update with table refresh and file full refresh", + update: pipelines.UpdateInfo{ + RefreshSelection: []string{"table1"}, + FullRefreshSelectionByPath: []string{"/Workspace/Shared/source.sql"}, + }, + expected: ` +Update for pipeline completed successfully. + +Pipeline configurations for this update: +• Refreshed [table1] +• Full refreshed tables defined in [/Workspace/Shared/source.sql] `, }, { diff --git a/cmd/pipelines/templates.go b/cmd/pipelines/templates.go index 3c471b366bb..f0bc50eedf1 100644 --- a/cmd/pipelines/templates.go +++ b/cmd/pipelines/templates.go @@ -29,7 +29,7 @@ Update end time: {{ .LastEventTime }} Pipeline configurations for this update: {{- if .Update.FullRefresh }} • All tables are fully refreshed -{{- else if and (eq (len .Update.RefreshSelection) 0) (eq (len .Update.FullRefreshSelection) 0) }} +{{- else if and (eq (len .Update.RefreshSelection) 0) (eq (len .Update.FullRefreshSelection) 0) (eq (len .Update.RefreshSelectionByPath) 0) (eq (len .Update.FullRefreshSelectionByPath) 0) }} • All tables are refreshed {{- else }} {{- if gt (len .Update.RefreshSelection) 0 }} @@ -38,6 +38,12 @@ Pipeline configurations for this update: {{- if gt (len .Update.FullRefreshSelection) 0 }} • Full refreshed [{{ join .Update.FullRefreshSelection ", " }}] {{- end }} +{{- if gt (len .Update.RefreshSelectionByPath) 0 }} +• Refreshed tables defined in [{{ join .Update.RefreshSelectionByPath ", " }}] +{{- end }} +{{- if gt (len .Update.FullRefreshSelectionByPath) 0 }} +• Full refreshed tables defined in [{{ join .Update.FullRefreshSelectionByPath ", " }}] +{{- end }} {{- end }} {{- if .Update.Cause }} • Update cause: {{ .Update.Cause }} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 98c52d1fc6c..a719aa911f3 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -222,7 +222,7 @@ type FakeWorkspace struct { JobRunOutputs map[int64]jobs.RunOutput JobRunIdempotency map[string]int64 Pipelines map[string]pipelines.GetPipelineResponse - PipelineUpdates map[string]bool + PipelineUpdates map[string]pipelines.UpdateInfo Monitors map[string]catalog.MonitorInfo DataQualityMonitors map[string]dataquality.Monitor Apps map[string]apps.App @@ -533,7 +533,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { JobRunIdempotency: map[string]int64{}, Grants: map[string][]catalog.PrivilegeAssignment{}, Pipelines: map[string]pipelines.GetPipelineResponse{}, - PipelineUpdates: map[string]bool{}, + PipelineUpdates: map[string]pipelines.UpdateInfo{}, Monitors: map[string]catalog.MonitorInfo{}, DataQualityMonitors: map[string]dataquality.Monitor{}, Apps: map[string]apps.App{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 8d7708af5af..fbd43d17013 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -674,7 +674,7 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("POST", "/api/2.0/pipelines/{pipeline_id}/updates", func(req Request) any { - return req.Workspace.PipelineStartUpdate(req.Vars["pipeline_id"]) + return req.Workspace.PipelineStartUpdate(req, req.Vars["pipeline_id"]) }) server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}/events", func(req Request) any { diff --git a/libs/testserver/pipelines.go b/libs/testserver/pipelines.go index 0aedc44c5d3..6960aebf350 100644 --- a/libs/testserver/pipelines.go +++ b/libs/testserver/pipelines.go @@ -143,7 +143,7 @@ func (s *FakeWorkspace) PipelineUpdate(req Request, pipelineId string) Response return Response{} } -func (s *FakeWorkspace) PipelineStartUpdate(pipelineId string) Response { +func (s *FakeWorkspace) PipelineStartUpdate(req Request, pipelineId string) Response { defer s.LockUnlock()() _, exists := s.Pipelines[pipelineId] @@ -154,8 +154,25 @@ func (s *FakeWorkspace) PipelineStartUpdate(pipelineId string) Response { } } + var start pipelines.StartUpdate + if err := json.Unmarshal(req.Body, &start); err != nil { + return Response{ + Body: fmt.Sprintf("cannot unmarshal request body: %s", err), + StatusCode: 400, + } + } + updateId := nextUUID() - s.PipelineUpdates[updateId] = true + // GetUpdate echoes selectors; the execution engine resolves paths to tables. + s.PipelineUpdates[updateId] = pipelines.UpdateInfo{ + UpdateId: updateId, + State: pipelines.UpdateInfoStateCompleted, + FullRefresh: start.FullRefresh, + RefreshSelection: start.RefreshSelection, + FullRefreshSelection: start.FullRefreshSelection, + RefreshSelectionByPath: start.RefreshSelectionByPath, + FullRefreshSelectionByPath: start.FullRefreshSelectionByPath, + } return Response{ Body: pipelines.StartUpdateResponse{ @@ -194,7 +211,7 @@ func (s *FakeWorkspace) PipelineGetUpdate(pipelineId, updateId string) Response } // Check if the update exists - _, updateExists := s.PipelineUpdates[updateId] + update, updateExists := s.PipelineUpdates[updateId] if !updateExists { return Response{ StatusCode: 404, @@ -204,10 +221,7 @@ func (s *FakeWorkspace) PipelineGetUpdate(pipelineId, updateId string) Response return Response{ Body: pipelines.GetUpdateResponse{ - Update: &pipelines.UpdateInfo{ - UpdateId: updateId, - State: pipelines.UpdateInfoStateCompleted, - }, + Update: &update, }, } } diff --git a/libs/testserver/pipelines_test.go b/libs/testserver/pipelines_test.go index 39114714082..6bb1dadfbdb 100644 --- a/libs/testserver/pipelines_test.go +++ b/libs/testserver/pipelines_test.go @@ -51,7 +51,7 @@ func TestPipelineCreate_AllowsSingleSegmentTargetSchemaName(t *testing.T) { func TestPipelineStartUpdate_HandlesNonExistentPipeline(t *testing.T) { workspace := NewFakeWorkspace("http://test", "dbapi123") - response := workspace.PipelineStartUpdate("non-existent-pipeline") + response := workspace.PipelineStartUpdate(Request{Body: []byte(`{}`)}, "non-existent-pipeline") assert.Equal(t, 404, response.StatusCode) assert.Contains(t, response.Body.(map[string]string)["message"], "The specified pipeline non-existent-pipeline was not found") } @@ -74,7 +74,7 @@ func TestPipelineStop_AfterUpdate(t *testing.T) { pipelineId := createTestPipeline(t, workspace) - startResponse := workspace.PipelineStartUpdate(pipelineId) + startResponse := workspace.PipelineStartUpdate(Request{Body: []byte(`{}`)}, pipelineId) assert.Equal(t, 0, startResponse.StatusCode) stopResponse := workspace.PipelineStop(pipelineId) @@ -85,3 +85,43 @@ func TestPipelineStop_AfterUpdate(t *testing.T) { assert.Equal(t, pipelineId, stopBody.PipelineId) assert.Equal(t, pipelines.PipelineStateIdle, stopBody.State) } + +func TestPipelineGetUpdatePreservesRefreshSelections(t *testing.T) { + workspace := NewFakeWorkspace("http://workspace.test", "dbapi123") + pipelineId := createTestPipeline(t, workspace) + tests := []struct { + name string + body string + want pipelines.UpdateInfo + }{ + { + name: "file selections", + body: `{"refresh_selection_by_path":["/Workspace/src/orders.py","/Workspace/src/returns.py"],"full_refresh_selection_by_path":["/Workspace/src/customers.sql"]}`, + want: pipelines.UpdateInfo{ + RefreshSelectionByPath: []string{"/Workspace/src/orders.py", "/Workspace/src/returns.py"}, + FullRefreshSelectionByPath: []string{"/Workspace/src/customers.sql"}, + }, + }, + { + name: "table selections", + body: `{"refresh_selection":["orders"],"full_refresh_selection":["customers"]}`, + want: pipelines.UpdateInfo{ + RefreshSelection: []string{"orders"}, + FullRefreshSelection: []string{"customers"}, + }, + }, + } + for i := range tests { + response := workspace.PipelineStartUpdate(Request{Body: []byte(tests[i].body)}, pipelineId) + require.Zero(t, response.StatusCode) + tests[i].want.UpdateId = response.Body.(pipelines.StartUpdateResponse).UpdateId + tests[i].want.State = pipelines.UpdateInfoStateCompleted + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + response := workspace.PipelineGetUpdate(pipelineId, tt.want.UpdateId) + require.Zero(t, response.StatusCode) + assert.Equal(t, &tt.want, response.Body.(pipelines.GetUpdateResponse).Update) + }) + } +} From 34271855a67755e721f69efca2557f00038add5a Mon Sep 17 00:00:00 2001 From: "chenyue.ma" Date: Mon, 9 Nov 2026 04:50:38 +0000 Subject: [PATCH 2/9] Address refresh selector review feedback --- .nextchanges/cli/pipeline-refresh-by-file.md | 2 +- .../run/refresh-flags/databricks.yml | 21 +- .../pipelines/run/refresh-flags/out.test.toml | 2 +- .../pipelines/run/refresh-flags/output.txt | 213 ++++++++---------- .../run/refresh-flags/pipeline_file.py | 9 +- acceptance/pipelines/run/refresh-flags/script | 58 ++++- .../refresh-flags/src/pipeline_notebook.sql | 6 + .../pipelines/run/refresh-flags/test.toml | 2 + bundle/bundle.go | 21 ++ .../mutator/resolve_job_run_file_triggers.go | 11 +- .../mutator/resolve_variable_references.go | 2 +- bundle/config/mutator/translate_paths.go | 10 +- bundle/run/pipeline_options.go | 8 +- .../run/pipeline_refresh.go | 61 +++-- .../run/pipeline_refresh_test.go | 40 +++- cmd/bundle/run.go | 6 + cmd/pipelines/run.go | 41 ++-- cmd/pipelines/run_test.go | 36 +++ 18 files changed, 347 insertions(+), 202 deletions(-) create mode 100644 acceptance/pipelines/run/refresh-flags/src/pipeline_notebook.sql rename cmd/pipelines/refresh.go => bundle/run/pipeline_refresh.go (61%) rename cmd/pipelines/refresh_test.go => bundle/run/pipeline_refresh_test.go (77%) diff --git a/.nextchanges/cli/pipeline-refresh-by-file.md b/.nextchanges/cli/pipeline-refresh-by-file.md index c5e25a11058..bf3a0452e79 100644 --- a/.nextchanges/cli/pipeline-refresh-by-file.md +++ b/.nextchanges/cli/pipeline-refresh-by-file.md @@ -1 +1 @@ -* Support source files in `pipelines run --refresh` and `--full-refresh` selectors. ([#7184](https://github.com/databricks/cli/pull/7184)) +* Support source files in `pipelines run` and `bundle run` with `--refresh` and `--full-refresh`. ([#7184](https://github.com/databricks/cli/pull/7184)) diff --git a/acceptance/pipelines/run/refresh-flags/databricks.yml b/acceptance/pipelines/run/refresh-flags/databricks.yml index 37e89920ad4..7fe4b3f27b5 100644 --- a/acceptance/pipelines/run/refresh-flags/databricks.yml +++ b/acceptance/pipelines/run/refresh-flags/databricks.yml @@ -1,10 +1,29 @@ bundle: name: test-pipeline-run-flags +variables: + unique_name: + description: Unique name for test resources + +workspace: + root_path: ~/.bundle/${bundle.name}/${var.unique_name} + +presets: + pipelines_development: true + resources: + schemas: + test_schema: + name: pipeline_refresh_${var.unique_name} + catalog_name: main pipelines: my_pipeline: - name: test-pipeline-flags + name: test-pipeline-flags-${var.unique_name} + catalog: main + schema: ${resources.schemas.test_schema.name} + serverless: true libraries: - file: path: pipeline_file.py + - notebook: + path: src/pipeline_notebook.sql diff --git a/acceptance/pipelines/run/refresh-flags/out.test.toml b/acceptance/pipelines/run/refresh-flags/out.test.toml index 18b1a88417e..c7c6f58ed6e 100644 --- a/acceptance/pipelines/run/refresh-flags/out.test.toml +++ b/acceptance/pipelines/run/refresh-flags/out.test.toml @@ -1 +1 @@ -Cloud = false +Cloud = true diff --git a/acceptance/pipelines/run/refresh-flags/output.txt b/acceptance/pipelines/run/refresh-flags/output.txt index b821445daa3..10ff83edfcf 100644 --- a/acceptance/pipelines/run/refresh-flags/output.txt +++ b/acceptance/pipelines/run/refresh-flags/output.txt @@ -1,85 +1,54 @@ >>> [CLI] pipelines deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files... Created pipelines.my_pipeline -Files: 2 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID] - -=== Running pipeline with --refresh flag and specific tables - ->>> [CLI] pipelines run --refresh table1,table2 -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][1] - -Update ID: [UUID][1] - -Update for pipeline completed successfully. - -Pipeline configurations for this update: -• Refreshed [table1, table2] +Created schemas.test_schema +Files: 3 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] +View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_refresh_[UNIQUE_NAME]?w=[WORKSPACE_ID] + +=== Initialize all tables with --full-refresh-all +>>> run_pipeline --full-refresh-all +• All tables are fully refreshed >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID][0]/updates", + "path": "/api/2.0/pipelines/[UUID]/updates", "body": { - "refresh_selection": [ - "table1", - "table2" - ] + "development": true, + "full_refresh": true } } -=== Running pipeline with --full-refresh-all flag - ->>> [CLI] pipelines run --full-refresh-all -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][2] - -Update ID: [UUID][2] - -Update for pipeline completed successfully. - -Pipeline configurations for this update: -• All tables are fully refreshed +=== Running pipeline with --refresh flag and specific tables +>>> run_pipeline --refresh table1,table2 +• Refreshed [table1, table2] >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID][0]/updates", + "path": "/api/2.0/pipelines/[UUID]/updates", "body": { - "full_refresh": true + "development": true, + "refresh_selection": [ + "table1", + "table2" + ] } } === Running pipeline with --full-refresh flag and specific tables - ->>> [CLI] pipelines run --full-refresh table1,table2 -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][3] - -Update ID: [UUID][3] - -Update for pipeline completed successfully. - -Pipeline configurations for this update: +>>> run_pipeline --full-refresh table1,table2 • Full refreshed [table1, table2] >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID][0]/updates", + "path": "/api/2.0/pipelines/[UUID]/updates", "body": { + "development": true, "full_refresh_selection": [ "table1", "table2" @@ -88,27 +57,16 @@ Pipeline configurations for this update: } === Running pipeline with --full-refresh flag and --refresh flag - ->>> [CLI] pipelines run --full-refresh table1,table2 --refresh table3,table4 -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][4] - -Update ID: [UUID][4] - -Update for pipeline completed successfully. - -Pipeline configurations for this update: +>>> run_pipeline --full-refresh table1,table2 --refresh table3,table4 • Refreshed [table3, table4] • Full refreshed [table1, table2] >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID][0]/updates", + "path": "/api/2.0/pipelines/[UUID]/updates", "body": { + "development": true, "full_refresh_selection": [ "table1", "table2" @@ -121,94 +79,123 @@ Pipeline configurations for this update: } === Refreshing tables defined in a source file ->>> [CLI] pipelines run --refresh pipeline_file.py -Recommendation: This command runs the last deployed version of the code +>>> run_pipeline --refresh pipeline_file.py +• Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py" + ] + } +} -Update ID: [UUID] +=== Fully refreshing tables defined in a source file +>>> run_pipeline --full-refresh pipeline_file.py +• Full refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] -Update for pipeline completed successfully. +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "full_refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py" + ] + } +} -Pipeline configurations for this update: -• Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files/pipeline_file.py] +=== Refreshing a source notebook +>>> run_pipeline --refresh src/pipeline_notebook.sql +• Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/src/pipeline_notebook] >>> print_requests.py //pipelines { "method": "POST", "path": "/api/2.0/pipelines/[UUID]/updates", "body": { + "development": true, "refresh_selection_by_path": [ - "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files/pipeline_file.py" + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/src/pipeline_notebook" ] } } -=== Fully refreshing tables defined in a source file ->>> [CLI] pipelines run --full-refresh pipeline_file.py -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] +=== Refreshing an absolute workspace path +>>> run_pipeline --refresh /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py +• Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] -Update ID: [UUID] +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py" + ] + } +} -Update for pipeline completed successfully. +=== Refreshing from a subdirectory +>>> withdir src run_pipeline --refresh ../pipeline_file.py +• Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] -Pipeline configurations for this update: -• Full refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files/pipeline_file.py] +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py" + ] + } +} +=== Bundle run supports both file flags >>> print_requests.py //pipelines { "method": "POST", "path": "/api/2.0/pipelines/[UUID]/updates", "body": { + "development": true, "full_refresh_selection_by_path": [ - "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files/pipeline_file.py" + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/src/pipeline_notebook" + ], + "refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py" ] } } === Rejecting missing sources and incompatible refresh modes >>> musterr [CLI] pipelines run --refresh ./missing.sql --no-wait -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - Error: invalid --refresh: source file "./missing.sql" not found; use a table name or an existing source file >>> musterr [CLI] pipelines run --refresh pipeline_file.py --full-refresh-all --no-wait -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - Error: cannot use --refresh or --full-refresh together with --full-refresh-all, these flags are mutually exclusive >>> musterr [CLI] pipelines run --full-refresh ./missing.sql --restart --no-wait -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - Error: invalid --full-refresh: source file "./missing.sql" not found; use a table name or an existing source file ->>> test ! -e [OUTPUT_DIR]/out.requests.txt +>>> print_requests.py //pipelines === Rejecting mixed selectors before restarting the pipeline >>> musterr [CLI] pipelines run --refresh table1,pipeline_file.py --restart --no-wait -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - Error: cannot mix table names and source files in --refresh; use only table names or only source files >>> musterr [CLI] pipelines run --full-refresh table2,pipeline_file.py --restart --no-wait -Recommendation: This command runs the last deployed version of the code +Error: cannot mix table names and source files in --full-refresh; use only table names or only source files -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. +>>> print_requests.py //pipelines -Error: cannot mix table names and source files in --full-refresh; use only table names or only source files +>>> musterr [CLI] bundle run my_pipeline --refresh table1,pipeline_file.py --restart --no-wait +Error: cannot mix table names and source files in --refresh; use only table names or only source files ->>> test ! -e [OUTPUT_DIR]/out.requests.txt +>>> print_requests.py //pipelines diff --git a/acceptance/pipelines/run/refresh-flags/pipeline_file.py b/acceptance/pipelines/run/refresh-flags/pipeline_file.py index aa0c44e3f6d..51dcf9ed5d7 100644 --- a/acceptance/pipelines/run/refresh-flags/pipeline_file.py +++ b/acceptance/pipelines/run/refresh-flags/pipeline_file.py @@ -1,7 +1,14 @@ import dlt from pyspark.sql import SparkSession +spark = SparkSession.builder.getOrCreate() + @dlt.table -def my_table(): +def table1(): return spark.range(10) + + +@dlt.table +def table2(): + return spark.range(20) diff --git a/acceptance/pipelines/run/refresh-flags/script b/acceptance/pipelines/run/refresh-flags/script index e2838af2063..6ad3daf72a4 100644 --- a/acceptance/pipelines/run/refresh-flags/script +++ b/acceptance/pipelines/run/refresh-flags/script @@ -1,37 +1,75 @@ +export BUNDLE_VAR_unique_name="$UNIQUE_NAME" +cleanup() { + $CLI bundle destroy --auto-approve &> LOG.cleanup +} +trap cleanup EXIT + +run_log="$PWD/LOG.run" + +# Cloud event logs and durations vary between runs. +run_pipeline() { + $CLI pipelines run "$@" &> "$run_log" + contains.py "completed successfully." < "$run_log" > "$run_log.checked" + grep -E '^• (All tables are |Refreshed |Full refreshed )' "$run_log" +} + trace $CLI pipelines deploy +remote=$($CLI bundle summary -o json | jq -r .workspace.file_path) rm "$OUT_REQUESTS" -title "Running pipeline with --refresh flag and specific tables" -trace $CLI pipelines run --refresh table1,table2 +title "Initialize all tables with --full-refresh-all" +trace run_pipeline --full-refresh-all trace print_requests.py //pipelines -title "Running pipeline with --full-refresh-all flag" -trace $CLI pipelines run --full-refresh-all +title "Running pipeline with --refresh flag and specific tables" +trace run_pipeline --refresh table1,table2 trace print_requests.py //pipelines title "Running pipeline with --full-refresh flag and specific tables" -trace $CLI pipelines run --full-refresh table1,table2 +trace run_pipeline --full-refresh table1,table2 trace print_requests.py //pipelines title "Running pipeline with --full-refresh flag and --refresh flag" -trace $CLI pipelines run --full-refresh table1,table2 --refresh table3,table4 +trace run_pipeline --full-refresh table1,table2 --refresh table3,table4 trace print_requests.py //pipelines title "Refreshing tables defined in a source file" -trace $CLI pipelines run --refresh pipeline_file.py +trace run_pipeline --refresh pipeline_file.py trace print_requests.py //pipelines title "Fully refreshing tables defined in a source file" -trace $CLI pipelines run --full-refresh pipeline_file.py +trace run_pipeline --full-refresh pipeline_file.py +trace print_requests.py //pipelines + +title "Refreshing a source notebook" +trace run_pipeline --refresh src/pipeline_notebook.sql +trace print_requests.py //pipelines + +title "Refreshing an absolute workspace path" +trace run_pipeline --refresh "$remote/pipeline_file.py" +trace print_requests.py //pipelines + +title "Refreshing from a subdirectory" +trace withdir src run_pipeline --refresh ../pipeline_file.py +trace print_requests.py //pipelines + +title "Bundle run supports both file flags" +trace $CLI bundle run my_pipeline --refresh pipeline_file.py --full-refresh src/pipeline_notebook.sql &> LOG.bundle trace print_requests.py //pipelines title "Rejecting missing sources and incompatible refresh modes" trace musterr $CLI pipelines run --refresh ./missing.sql --no-wait trace musterr $CLI pipelines run --refresh pipeline_file.py --full-refresh-all --no-wait trace musterr $CLI pipelines run --full-refresh ./missing.sql --restart --no-wait -trace test ! -e "$OUT_REQUESTS" +touch "$OUT_REQUESTS" +trace print_requests.py //pipelines title "Rejecting mixed selectors before restarting the pipeline" trace musterr $CLI pipelines run --refresh table1,pipeline_file.py --restart --no-wait trace musterr $CLI pipelines run --full-refresh table2,pipeline_file.py --restart --no-wait -trace test ! -e "$OUT_REQUESTS" +touch "$OUT_REQUESTS" +trace print_requests.py //pipelines + +trace musterr $CLI bundle run my_pipeline --refresh table1,pipeline_file.py --restart --no-wait +touch "$OUT_REQUESTS" +trace print_requests.py //pipelines diff --git a/acceptance/pipelines/run/refresh-flags/src/pipeline_notebook.sql b/acceptance/pipelines/run/refresh-flags/src/pipeline_notebook.sql new file mode 100644 index 00000000000..be956582f1c --- /dev/null +++ b/acceptance/pipelines/run/refresh-flags/src/pipeline_notebook.sql @@ -0,0 +1,6 @@ +-- Databricks notebook source +CREATE OR REFRESH MATERIALIZED VIEW table3 AS SELECT 3 AS id; + +-- COMMAND ---------- + +CREATE OR REFRESH MATERIALIZED VIEW table4 AS SELECT 4 AS id; diff --git a/acceptance/pipelines/run/refresh-flags/test.toml b/acceptance/pipelines/run/refresh-flags/test.toml index 159efe02696..a86fb37aa56 100644 --- a/acceptance/pipelines/run/refresh-flags/test.toml +++ b/acceptance/pipelines/run/refresh-flags/test.toml @@ -1 +1,3 @@ +Cloud = true RecordRequests = true +Env.MSYS_NO_PATHCONV = "1" diff --git a/bundle/bundle.go b/bundle/bundle.go index cb615987468..ff22ce5d4a4 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -397,6 +397,27 @@ func (b *Bundle) InternalDir(ctx context.Context) (string, error) { return dir, nil } +// GetSyncFiles lists files matching the bundle sync configuration. +func (b *Bundle) GetSyncFiles(ctx context.Context) ([]fileset.File, error) { + includes, err := b.GetSyncIncludePatterns(ctx) + if err != nil { + return nil, err + } + list, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, includes, b.Config.Sync.Exclude) + if err != nil { + return nil, err + } + return list.Files(ctx) +} + +// RemoteFilePath returns the initialized sync destination. +func (b *Bundle) RemoteFilePath() string { + if b.IsSourceLinkedDeployment() { + return filepath.ToSlash(b.SyncRootPath) + } + return b.Config.Workspace.FilePath +} + // GetSyncIncludePatterns returns a list of user defined includes // And also adds InternalDir folder to include list for sync command // so this folder is always synced diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 35cffd0dcbe..32dd9c90ab7 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -15,7 +15,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - libsync "github.com/databricks/cli/libs/sync" ) // Every diagnostic below is reported against an on_file_change entry. @@ -85,15 +84,7 @@ func listSyncableRelPaths(ctx context.Context, b *bundle.Bundle) ([]string, erro // additionally drops files whose notebook type it cannot determine, such as a // malformed .ipynb. Those stay watched on purpose - editing one is exactly when the // run should re-fire - so a pattern can hash a file that this deploy will not upload. - includes, err := b.GetSyncIncludePatterns(ctx) - if err != nil { - return nil, err - } - fl, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, includes, b.Config.Sync.Exclude) - if err != nil { - return nil, err - } - files, err := fl.Files(ctx) + files, err := b.GetSyncFiles(ctx) if err != nil { return nil, err } diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index a5e0d3bf086..b8245569981 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -118,7 +118,7 @@ func ResolveVolumePathReferencesOnlyResources() bundle.Mutator { func lookup(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) { if b.IsSourceLinkedDeployment() { if path.String() == "workspace.file_path" { - return dyn.V(b.SyncRootPath), nil + return dyn.V(b.RemoteFilePath()), nil } } // Future opportunity: if we lookup this path in both the given root diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 956d010ec95..a2a32d14c06 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -15,7 +15,6 @@ import ( "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/notebook" @@ -320,8 +319,7 @@ func (t *translateContext) rewriteValue(ctx context.Context, p dyn.Path, v dyn.V } func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContext, translations []func(context.Context, dyn.Value) (dyn.Value, error)) diag.Diagnostics { - switch { - case b.IsImmutableFolder(): + if b.IsImmutableFolder() { // Reject an explicit workspace.file_path: immutable bundles set it // automatically to the content-addressed snapshot location. A user-supplied // value would be silently discarded during path translation, so we error early. @@ -332,12 +330,8 @@ func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContex Locations: []diag.Location{loc}, }} } - t.remoteRoot = resources.SnapshotFullPathRef + "/files" - case t.b.IsSourceLinkedDeployment(): - t.remoteRoot = t.b.SyncRootPath - default: - t.remoteRoot = t.b.Config.Workspace.FilePath } + t.remoteRoot = b.RemoteFilePath() err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { var err error diff --git a/bundle/run/pipeline_options.go b/bundle/run/pipeline_options.go index 164e8a2de61..e6a2a6753f9 100644 --- a/bundle/run/pipeline_options.go +++ b/bundle/run/pipeline_options.go @@ -15,7 +15,7 @@ type PipelineOptions struct { // Perform a full graph update. RefreshAll bool - // List of tables to update. + // List of tables or source files to update. Refresh []string RefreshByPath []string @@ -23,7 +23,7 @@ type PipelineOptions struct { // Perform a full graph reset and recompute. FullRefreshAll bool - // List of tables to reset and recompute. + // List of tables or source files to reset and recompute. FullRefresh []string FullRefreshByPath []string @@ -34,9 +34,9 @@ type PipelineOptions struct { func (o *PipelineOptions) Define(fs *flag.FlagSet) { fs.BoolVar(&o.RefreshAll, "refresh-all", false, "Perform a full graph update.") - fs.StringSliceVar(&o.Refresh, "refresh", nil, "List of tables to update.") + fs.StringSliceVar(&o.Refresh, "refresh", nil, "List of tables or source files to update. Local paths are relative to the current directory.") fs.BoolVar(&o.FullRefreshAll, "full-refresh-all", false, "Perform a full graph reset and recompute.") - fs.StringSliceVar(&o.FullRefresh, "full-refresh", nil, "List of tables to reset and recompute.") + fs.StringSliceVar(&o.FullRefresh, "full-refresh", nil, "List of tables or source files to reset and recompute. Local paths are relative to the current directory.") fs.BoolVar(&o.ValidateOnly, "validate-only", false, "Perform an update to validate graph correctness.") } diff --git a/cmd/pipelines/refresh.go b/bundle/run/pipeline_refresh.go similarity index 61% rename from cmd/pipelines/refresh.go rename to bundle/run/pipeline_refresh.go index d26da6e33a8..997579f8e44 100644 --- a/cmd/pipelines/refresh.go +++ b/bundle/run/pipeline_refresh.go @@ -1,4 +1,4 @@ -package pipelines +package run import ( "context" @@ -11,14 +11,36 @@ import ( "strings" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/notebook" "github.com/databricks/cli/libs/sync" ) -// resolveRefreshSelectors separates table names from source files and translates local files to workspace paths. -func resolveRefreshSelectors(ctx context.Context, b *bundle.Bundle, selectors []string) (tables, paths []string, err error) { - var remoteNames map[string]string +// ResolveRefreshSelectors resolves source files to workspace paths. +func (o *PipelineOptions) ResolveRefreshSelectors(ctx context.Context, b *bundle.Bundle) error { + r := refreshSelectorResolver{bundle: b} + var err error + o.Refresh, o.RefreshByPath, err = r.resolve(ctx, o.Refresh) + if err != nil { + return fmt.Errorf("invalid --refresh: %w", err) + } + o.FullRefresh, o.FullRefreshByPath, err = r.resolve(ctx, o.FullRefresh) + if err != nil { + return fmt.Errorf("invalid --full-refresh: %w", err) + } + // Validate before Restart stops an active update. + return o.Validate(nil) +} + +type refreshSelectorResolver struct { + bundle *bundle.Bundle + remoteNames map[string]string +} + +func (r *refreshSelectorResolver) resolve(ctx context.Context, selectors []string) (tables, paths []string, err error) { + b := r.bundle for _, selector := range selectors { + // /Workspace/ distinguishes remote paths from local paths like /Users/... on macOS. if strings.HasPrefix(selector, "/Workspace/") { paths = append(paths, selector) continue @@ -58,23 +80,26 @@ func resolveRefreshSelectors(ctx context.Context, b *bundle.Bundle, selectors [] if !filepath.IsLocal(rel) { return nil, nil, fmt.Errorf("source file %q is outside the bundle sync root", selector) } - if remoteNames == nil { - remoteNames, err = refreshSourcePaths(ctx, b) + if r.remoteNames == nil { + // Refresh uses the deployed snapshot. + if b.IsImmutableFolder() { + if err := b.DeploymentBundle.ResolveConfigAgainstState(&b.Config, dyn.MustPathFromString("workspace.file_path")); err != nil { + return nil, nil, err + } + } + r.remoteNames, err = refreshSourcePaths(ctx, b) if err != nil { return nil, nil, err } } - remoteName, ok := remoteNames[filepath.ToSlash(rel)] + remoteName, ok := r.remoteNames[filepath.ToSlash(rel)] if !ok { if _, _, err := notebook.DetectWithFS(b.SyncRoot, filepath.ToSlash(rel)); err != nil { return nil, nil, fmt.Errorf("cannot identify source file %q: %w", selector, err) } - return nil, nil, fmt.Errorf("source file %q is excluded from bundle sync; include it and run 'databricks pipelines deploy' first", selector) - } - remoteRoot := b.Config.Workspace.FilePath - if b.IsSourceLinkedDeployment() { - remoteRoot = filepath.ToSlash(b.SyncRootPath) + return nil, nil, fmt.Errorf("source file %q is excluded from bundle sync; include it and deploy first", selector) } + remoteRoot := b.RemoteFilePath() if strings.Contains(remoteRoot, "${") { return nil, nil, fmt.Errorf("cannot resolve the deployed path for %q; use its absolute /Workspace/ path", selector) } @@ -83,17 +108,9 @@ func resolveRefreshSelectors(ctx context.Context, b *bundle.Bundle, selectors [] return tables, paths, nil } -// refreshSourcePaths uses the sync mapping so notebook names and sync exclusions match deployment. +// refreshSourcePaths maps local files to their deployed names. func refreshSourcePaths(ctx context.Context, b *bundle.Bundle) (map[string]string, error) { - includes, err := b.GetSyncIncludePatterns(ctx) - if err != nil { - return nil, err - } - list, err := sync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, includes, b.Config.Sync.Exclude) - if err != nil { - return nil, err - } - files, err := list.Files(ctx) + files, err := b.GetSyncFiles(ctx) if err != nil { return nil, err } diff --git a/cmd/pipelines/refresh_test.go b/bundle/run/pipeline_refresh_test.go similarity index 77% rename from cmd/pipelines/refresh_test.go rename to bundle/run/pipeline_refresh_test.go index eb1824a5890..27b9c098bd1 100644 --- a/cmd/pipelines/refresh_test.go +++ b/bundle/run/pipeline_refresh_test.go @@ -1,4 +1,4 @@ -package pipelines +package run import ( "os" @@ -8,6 +8,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -74,7 +76,7 @@ func TestResolveRefreshSelectors(t *testing.T) { t.Chdir(root) b := refreshTestBundle(root) b.Config.Sync.Exclude = []string{"src/excluded.sql"} - tables, paths, err := resolveRefreshSelectors(t.Context(), b, tt.selectors) + tables, paths, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), tt.selectors) if tt.err != "" { require.ErrorContains(t, err, tt.err) return @@ -108,7 +110,7 @@ func TestResolveRefreshSelectorsFromSubdirectory(t *testing.T) { b := refreshTestBundle(root) b.BundleRootPath = filepath.Join(root, "project") for _, selector := range []string{"../../shared.sql", filepath.Join(root, "shared.sql")} { - tables, paths, err := resolveRefreshSelectors(t.Context(), b, []string{selector}) + tables, paths, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), []string{selector}) require.NoError(t, err) assert.Empty(t, tables) assert.Equal(t, []string{"/Workspace/Shared/project/files/shared.sql"}, paths) @@ -120,7 +122,7 @@ func TestResolveRefreshSelectorsOutsideSyncRoot(t *testing.T) { require.NoError(t, os.Mkdir(filepath.Join(root, "project"), 0o700)) require.NoError(t, os.WriteFile(filepath.Join(root, "outside.sql"), []byte("SELECT 1;\n"), 0o600)) t.Chdir(filepath.Join(root, "project")) - _, _, err := resolveRefreshSelectors(t.Context(), refreshTestBundle(filepath.Join(root, "project")), []string{"../outside.sql"}) + _, _, err := (&refreshSelectorResolver{bundle: refreshTestBundle(filepath.Join(root, "project"))}).resolve(t.Context(), []string{"../outside.sql"}) require.ErrorContains(t, err, "is outside the bundle sync root") } @@ -148,7 +150,7 @@ func TestResolveRefreshSelectorsDeploymentMode(t *testing.T) { b := refreshTestBundle(root) b.Config.Workspace.SyncMode = tt.mode b.Config.Presets.SourceLinkedDeployment = tt.sourceLinked - tables, paths, err := resolveRefreshSelectors(t.Context(), b, []string{"query.sql"}) + tables, paths, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), []string{"query.sql"}) require.NoError(t, err) assert.Empty(t, tables) assert.Equal(t, []string{tt.want}, paths) @@ -165,6 +167,32 @@ func TestResolveRefreshSelectorsUnresolvedWorkspacePath(t *testing.T) { b.Config.Workspace.SyncMode = config.SyncModeImmutable enabled := true b.Config.Presets.SourceLinkedDeployment = &enabled - _, _, err := resolveRefreshSelectors(t.Context(), b, []string{"query.sql"}) + b.DeploymentBundle.StateDB.OpenWithData("state", dstate.Database{}) + _, _, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), []string{"query.sql"}) require.ErrorContains(t, err, "use its absolute /Workspace/ path") } + +func TestResolveRefreshSelectorsDeployedSnapshot(t *testing.T) { + root := t.TempDir() + for _, name := range []string{"query.sql", "full_query.sql"} { + require.NoError(t, os.WriteFile(filepath.Join(root, name), []byte("SELECT 2;\n"), 0o600)) + } + t.Chdir(root) + b := refreshTestBundle(root) + b.Config.Workspace.SyncMode = config.SyncModeImmutable + b.Config.Workspace.FilePath = resources.SnapshotFullPathRef + "/files" + b.DeploymentBundle.StateDB.OpenWithData("state", dstate.Database{ + State: map[string]dstate.ResourceEntry{ + resources.SnapshotKey: { + ID: "deployed-snapshot", + State: []byte(`{"full_path":"/Workspace/Shared/snapshots/deployed/snapshot"}`), + }, + }, + }) + opts := PipelineOptions{Refresh: []string{"query.sql"}, FullRefresh: []string{"full_query.sql"}} + require.NoError(t, opts.ResolveRefreshSelectors(t.Context(), b)) + assert.Empty(t, opts.Refresh) + assert.Empty(t, opts.FullRefresh) + assert.Equal(t, []string{"/Workspace/Shared/snapshots/deployed/snapshot/files/query.sql"}, opts.RefreshByPath) + assert.Equal(t, []string{"/Workspace/Shared/snapshots/deployed/snapshot/files/full_query.sql"}, opts.FullRefreshByPath) +} diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 17199a6cabf..d27c3a12695 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -206,6 +206,12 @@ Example usage: return err } + if ref.Description.SingularName == "pipeline" { + if err := runOptions.Pipeline.ResolveRefreshSelectors(ctx, b); err != nil { + return err + } + } + runOptions.NoWait = noWait var runOutput output.RunOutput if restart { diff --git a/cmd/pipelines/run.go b/cmd/pipelines/run.go index 702a8392edf..f86fedb3b70 100644 --- a/cmd/pipelines/run.go +++ b/cmd/pipelines/run.go @@ -17,6 +17,7 @@ import ( bundleresources "github.com/databricks/cli/bundle/resources" "github.com/databricks/cli/bundle/run" bundlerunoutput "github.com/databricks/cli/bundle/run/output" + "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdgroup" @@ -179,15 +180,18 @@ func fetchAndDisplayPipelineUpdate(ctx context.Context, w *databricks.WorkspaceC latestUpdate := *getUpdateResponse.Update - params := &PipelineEventsQueryParams{ - Filter: fmt.Sprintf("update_id='%s' AND event_type='update_progress'", updateId), - OrderBy: "timestamp asc", - } - - events, err := fetchAllPipelineEvents(ctx, w, pipelineId, params) + // The public events API rejects event_type filters. + events, err := w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ + PipelineId: pipelineId, + Filter: fmt.Sprintf("update_id='%s'", updateId), + OrderBy: []string{"timestamp asc"}, + }) if err != nil { return err } + events = slices.DeleteFunc(events, func(event pipelines.PipelineEvent) bool { + return event.EventType != "update_progress" + }) err = displayPipelineUpdate(ctx, latestUpdate, pipelineId, events) if err != nil { @@ -264,6 +268,10 @@ a file, or quote table identifiers with backticks to select an ambiguous table.` ctx := logdiag.InitContext(cmd.Context()) cmd.SetContext(ctx) + runOptions := run.Options{ + Pipeline: run.PipelineOptions{Refresh: refresh, FullRefresh: fullRefresh, FullRefreshAll: fullRefreshAll}, + NoWait: noWait, + } var key string b, err := utils.ProcessBundle(cmd, utils.ProcessOptions{ PostInitFunc: func(ctx context.Context, b *bundle.Bundle) error { @@ -271,6 +279,9 @@ a file, or quote table identifiers with backticks to select an ambiguous table.` key, _, err = resolveRunArgument(ctx, b, args) return err }, + PostStateFunc: func(ctx context.Context, b *bundle.Bundle, _ *statemgmt.StateDesc) error { + return runOptions.Pipeline.ResolveRefreshSelectors(ctx, b) + }, ErrorOnEmptyState: true, SkipInitContext: true, }) @@ -285,24 +296,6 @@ a file, or quote table identifiers with backticks to select an ambiguous table.` return err } - runOptions := run.Options{ - Pipeline: run.PipelineOptions{ - FullRefreshAll: fullRefreshAll, - }, - NoWait: noWait, - } - runOptions.Pipeline.Refresh, runOptions.Pipeline.RefreshByPath, err = resolveRefreshSelectors(ctx, b, refresh) - if err != nil { - return fmt.Errorf("invalid --refresh: %w", err) - } - runOptions.Pipeline.FullRefresh, runOptions.Pipeline.FullRefreshByPath, err = resolveRefreshSelectors(ctx, b, fullRefresh) - if err != nil { - return fmt.Errorf("invalid --full-refresh: %w", err) - } - if err := runOptions.Pipeline.Validate(nil); err != nil { - return err - } - var runOutput bundlerunoutput.RunOutput if restart { runOutput, err = runner.Restart(ctx, &runOptions) diff --git a/cmd/pipelines/run_test.go b/cmd/pipelines/run_test.go index 2c910669e34..246380a8c8e 100644 --- a/cmd/pipelines/run_test.go +++ b/cmd/pipelines/run_test.go @@ -5,10 +5,46 @@ import ( "time" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestFetchAndDisplayPipelineUpdateFiltersProgressAcrossPages(t *testing.T) { + server := testserver.New(t) + server.Handle("GET", "/api/2.0/pipelines/pipeline-1/updates/update-1", func(req testserver.Request) any { + return pipelines.GetUpdateResponse{Update: &pipelines.UpdateInfo{UpdateId: "update-1"}} + }) + server.Handle("GET", "/api/2.0/pipelines/pipeline-1/events", func(req testserver.Request) any { + if req.URL.Query().Get("page_token") == "" { + assert.Equal(t, "update_id='update-1'", req.URL.Query().Get("filter")) + return pipelines.ListPipelineEventsResponse{ + NextPageToken: "next-page", + Events: []pipelines.PipelineEvent{ + {EventType: "update_progress", Timestamp: "2026-01-01T00:00:00Z", Message: "Update is WAITING_FOR_RESOURCES."}, + {EventType: "flow_progress", Timestamp: "2026-01-01T00:00:05Z", Message: "Flow is PLANNING."}, + }, + } + } + assert.Equal(t, "next-page", req.URL.Query().Get("page_token")) + return pipelines.ListPipelineEventsResponse{ + Events: []pipelines.PipelineEvent{ + {EventType: "update_progress", Timestamp: "2026-01-01T00:00:10Z", Message: "Update is COMPLETED."}, + }, + } + }) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "test-token"}) + require.NoError(t, err) + ctx, stdout := cmdio.NewTestContextWithStdout(t.Context()) + + err = fetchAndDisplayPipelineUpdate(ctx, w, "pipeline-1", "update-1") + require.NoError(t, err) + assert.Contains(t, stdout.String(), "WAITING_FOR_RESOURCES") + assert.Contains(t, stdout.String(), "10.0s") +} + func TestDisplayPipelineUpdate(t *testing.T) { tests := []struct { name string From 874234fd403f96c4e230986936d04355646071c6 Mon Sep 17 00:00:00 2001 From: "chenyue.ma" Date: Mon, 9 Nov 2026 06:26:41 +0000 Subject: [PATCH 3/9] Fix Windows refresh acceptance paths --- acceptance/pipelines/run/refresh-flags/script | 2 +- acceptance/pipelines/run/refresh-flags/test.toml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/acceptance/pipelines/run/refresh-flags/script b/acceptance/pipelines/run/refresh-flags/script index 6ad3daf72a4..2df84af88c7 100644 --- a/acceptance/pipelines/run/refresh-flags/script +++ b/acceptance/pipelines/run/refresh-flags/script @@ -8,7 +8,7 @@ run_log="$PWD/LOG.run" # Cloud event logs and durations vary between runs. run_pipeline() { - $CLI pipelines run "$@" &> "$run_log" + MSYS_NO_PATHCONV=1 $CLI pipelines run "$@" &> "$run_log" contains.py "completed successfully." < "$run_log" > "$run_log.checked" grep -E '^• (All tables are |Refreshed |Full refreshed )' "$run_log" } diff --git a/acceptance/pipelines/run/refresh-flags/test.toml b/acceptance/pipelines/run/refresh-flags/test.toml index a86fb37aa56..ff9f96e2553 100644 --- a/acceptance/pipelines/run/refresh-flags/test.toml +++ b/acceptance/pipelines/run/refresh-flags/test.toml @@ -1,3 +1,2 @@ Cloud = true RecordRequests = true -Env.MSYS_NO_PATHCONV = "1" From cd32f1823102c8572b84d33f70733985cadd5811 Mon Sep 17 00:00:00 2001 From: "chenyue.ma" Date: Mon, 9 Nov 2026 06:55:56 +0000 Subject: [PATCH 4/9] Update bundle run refresh help snapshot --- acceptance/bundle/help/bundle-run/output.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt index a6b453c8834..62857b1a5c9 100644 --- a/acceptance/bundle/help/bundle-run/output.txt +++ b/acceptance/bundle/help/bundle-run/output.txt @@ -54,9 +54,9 @@ Job Task Flags: --sql-params stringToString A map from keys to values for jobs with SQL tasks. (default []) Pipeline Flags: - --full-refresh strings List of tables to reset and recompute. + --full-refresh strings List of tables or source files to reset and recompute. Local paths are relative to the current directory. --full-refresh-all Perform a full graph reset and recompute. - --refresh strings List of tables to update. + --refresh strings List of tables or source files to update. Local paths are relative to the current directory. --refresh-all Perform a full graph update. --validate-only Perform an update to validate graph correctness. From 765b5573fbf4697931ecb533d500f78d131b84fd Mon Sep 17 00:00:00 2001 From: "chenyue.ma" Date: Tue, 10 Nov 2026 06:12:03 +0000 Subject: [PATCH 5/9] Use explicit space-separated pipeline refresh selectors --- .nextchanges/cli/pipeline-refresh-by-file.md | 2 +- acceptance/bundle/help/bundle-run/output.txt | 18 +- .../bundle/run/refresh-flags/output.txt | 81 ++++++--- acceptance/bundle/run/refresh-flags/script | 22 ++- .../pipelines/run/refresh-flags/output.txt | 157 ++++++++++++++++-- acceptance/pipelines/run/refresh-flags/script | 61 +++++-- bundle/run/pipeline_options.go | 24 ++- bundle/run/pipeline_options_test.go | 36 +++- bundle/run/pipeline_refresh.go | 103 ++++++++++-- bundle/run/pipeline_refresh_test.go | 99 +++++++++-- cmd/bundle/run.go | 8 + cmd/pipelines/run.go | 25 ++- 12 files changed, 525 insertions(+), 111 deletions(-) diff --git a/.nextchanges/cli/pipeline-refresh-by-file.md b/.nextchanges/cli/pipeline-refresh-by-file.md index bf3a0452e79..cd3decf3d16 100644 --- a/.nextchanges/cli/pipeline-refresh-by-file.md +++ b/.nextchanges/cli/pipeline-refresh-by-file.md @@ -1 +1 @@ -* Support source files in `pipelines run` and `bundle run` with `--refresh` and `--full-refresh`. ([#7184](https://github.com/databricks/cli/pull/7184)) +* Support `file:` selectors in `pipelines run` and pipeline `bundle run` with `--refresh` and `--full-refresh`. Both flags now accept space-separated selectors inside quotes or repeated flags; bare names select tables and `table:` is optional. Replace comma-separated lists such as `--refresh orders,payments` with `--refresh "orders payments"`. Use backticks for names or paths containing spaces or commas, and double backticks to escape a literal backtick. ([#7184](https://github.com/databricks/cli/pull/7184)) diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt index 62857b1a5c9..a5c846fa795 100644 --- a/acceptance/bundle/help/bundle-run/output.txt +++ b/acceptance/bundle/help/bundle-run/output.txt @@ -19,6 +19,14 @@ parameter names. If the specified job does not use job parameters and the job has a Python file task or a Python wheel task, the second example applies. +For pipelines, --refresh and --full-refresh accept space-separated table: +or file: selectors inside quotes, or repeated flags. Bare names select tables. +For example: --refresh "orders payments" or --refresh "file:./src/a.py file:./src/b.py". +Use backticks around names or paths containing spaces or commas, and double a +backtick to escape it. In Bash and zsh, put values containing backticks in single quotes. +Comma-separated lists are no longer supported. +Deploy local file changes before running. Each flag accepts either tables or files. + --------------------------------------------------------- You can also use the bundle run command to execute scripts / commands in the same @@ -54,11 +62,11 @@ Job Task Flags: --sql-params stringToString A map from keys to values for jobs with SQL tasks. (default []) Pipeline Flags: - --full-refresh strings List of tables or source files to reset and recompute. Local paths are relative to the current directory. - --full-refresh-all Perform a full graph reset and recompute. - --refresh strings List of tables or source files to update. Local paths are relative to the current directory. - --refresh-all Perform a full graph update. - --validate-only Perform an update to validate graph correctness. + --full-refresh selectors Space-separated table: or file: selectors to reset and recompute. Bare names select tables. Repeatable. + --full-refresh-all Perform a full graph reset and recompute. + --refresh selectors Space-separated table: or file: selectors to update. Bare names select tables. Repeatable. + --refresh-all Perform a full graph update. + --validate-only Perform an update to validate graph correctness. Flags: -h, --help help for run diff --git a/acceptance/bundle/run/refresh-flags/output.txt b/acceptance/bundle/run/refresh-flags/output.txt index f5b077ea23d..1832375e9e7 100644 --- a/acceptance/bundle/run/refresh-flags/output.txt +++ b/acceptance/bundle/run/refresh-flags/output.txt @@ -6,16 +6,15 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Running pipeline with --refresh flag and specific tables +>>> [CLI] bundle run my_pipeline --refresh table1 table2 +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] ->>> [CLI] bundle run my_pipeline --refresh table1,table2 -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][1] - -Update ID: [UUID][1] +Update ID: [UUID] >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID][0]/updates", + "path": "/api/2.0/pipelines/[UUID]/updates", "body": { "refresh_selection": [ "table1", @@ -25,32 +24,30 @@ Update ID: [UUID][1] } === Running pipeline with --full-refresh-all flag - >>> [CLI] bundle run my_pipeline --full-refresh-all -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][2] +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] -Update ID: [UUID][2] +Update ID: [UUID] >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID][0]/updates", + "path": "/api/2.0/pipelines/[UUID]/updates", "body": { "full_refresh": true } } === Running pipeline with --full-refresh flag and specific tables +>>> [CLI] bundle run my_pipeline --full-refresh table1 table2 +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] ->>> [CLI] bundle run my_pipeline --full-refresh table1,table2 -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][3] - -Update ID: [UUID][3] +Update ID: [UUID] >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID][0]/updates", + "path": "/api/2.0/pipelines/[UUID]/updates", "body": { "full_refresh_selection": [ "table1", @@ -60,16 +57,15 @@ Update ID: [UUID][3] } === Running pipeline with --full-refresh flag and --refresh flag +>>> [CLI] bundle run my_pipeline --full-refresh table1 table2 --refresh table3 table4 +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] ->>> [CLI] bundle run my_pipeline --full-refresh table1,table2 --refresh table3,table4 -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][4] - -Update ID: [UUID][4] +Update ID: [UUID] >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID][0]/updates", + "path": "/api/2.0/pipelines/[UUID]/updates", "body": { "full_refresh_selection": [ "table1", @@ -81,3 +77,50 @@ Update ID: [UUID][4] ] } } + +=== Repeated refresh flags +>>> [CLI] bundle run my_pipeline --refresh table1 --refresh table:table2 +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] + +Update ID: [UUID] + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "refresh_selection": [ + "table1", + "table2" + ] + } +} + +=== Quoted table selectors +>>> [CLI] bundle run my_pipeline --refresh table:`table1` `table2` +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] + +Update ID: [UUID] + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "refresh_selection": [ + "table1", + "table2" + ] + } +} + +=== Invalid refresh syntax is rejected before restart>>> musterr [CLI] bundle run my_pipeline --refresh table1,table2 --restart --no-wait +Error: invalid argument "table1,table2" for "--refresh" flag: comma-separated selectors are no longer supported; separate selectors with spaces inside quotes, or repeat the flag; use backticks for literal commas +>>> musterr [CLI] bundle run my_pipeline --full-refresh table1,table2 --restart --no-wait +Error: invalid argument "table1,table2" for "--full-refresh" flag: comma-separated selectors are no longer supported; separate selectors with spaces inside quotes, or repeat the flag; use backticks for literal commas +>>> musterr [CLI] bundle run my_pipeline --refresh table:`table1 --restart --no-wait +Error: invalid argument "table:`table1" for "--refresh" flag: unclosed backtick in refresh selector +>>> musterr [CLI] bundle run my_pipeline --refresh --restart --no-wait +Error: invalid argument "" for "--refresh" flag: expected at least one table or file selector + +>>> print_requests.py //pipelines diff --git a/acceptance/bundle/run/refresh-flags/script b/acceptance/bundle/run/refresh-flags/script index 7c788c42239..eaa3ee2e9fe 100644 --- a/acceptance/bundle/run/refresh-flags/script +++ b/acceptance/bundle/run/refresh-flags/script @@ -2,7 +2,7 @@ trace $CLI bundle deploy rm "$OUT_REQUESTS" title "Running pipeline with --refresh flag and specific tables" -trace $CLI bundle run my_pipeline --refresh table1,table2 +trace $CLI bundle run my_pipeline --refresh "table1 table2" trace print_requests.py //pipelines title "Running pipeline with --full-refresh-all flag" @@ -10,9 +10,25 @@ trace $CLI bundle run my_pipeline --full-refresh-all trace print_requests.py //pipelines title "Running pipeline with --full-refresh flag and specific tables" -trace $CLI bundle run my_pipeline --full-refresh table1,table2 +trace $CLI bundle run my_pipeline --full-refresh "table1 table2" trace print_requests.py //pipelines title "Running pipeline with --full-refresh flag and --refresh flag" -trace $CLI bundle run my_pipeline --full-refresh table1,table2 --refresh table3,table4 +trace $CLI bundle run my_pipeline --full-refresh "table1 table2" --refresh "table3 table4" +trace print_requests.py //pipelines + +title "Repeated refresh flags" +trace $CLI bundle run my_pipeline --refresh table1 --refresh table:table2 +trace print_requests.py //pipelines + +title "Quoted table selectors" +trace $CLI bundle run my_pipeline --refresh 'table:`table1` `table2`' +trace print_requests.py //pipelines + +title "Invalid refresh syntax is rejected before restart" +trace musterr $CLI bundle run my_pipeline --refresh table1,table2 --restart --no-wait 2>&1 | grep -E '^(>>> |Error:)' +trace musterr $CLI bundle run my_pipeline --full-refresh table1,table2 --restart --no-wait 2>&1 | grep -E '^(>>> |Error:)' +trace musterr $CLI bundle run my_pipeline --refresh 'table:`table1' --restart --no-wait 2>&1 | grep -E '^(>>> |Error:)' +trace musterr $CLI bundle run my_pipeline --refresh '' --restart --no-wait 2>&1 | grep -E '^(>>> |Error:)' +touch "$OUT_REQUESTS" trace print_requests.py //pipelines diff --git a/acceptance/pipelines/run/refresh-flags/output.txt b/acceptance/pipelines/run/refresh-flags/output.txt index 10ff83edfcf..6bdc78ef33e 100644 --- a/acceptance/pipelines/run/refresh-flags/output.txt +++ b/acceptance/pipelines/run/refresh-flags/output.txt @@ -23,7 +23,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Running pipeline with --refresh flag and specific tables ->>> run_pipeline --refresh table1,table2 +>>> run_pipeline --refresh table1 table2 • Refreshed [table1, table2] >>> print_requests.py //pipelines @@ -40,7 +40,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Running pipeline with --full-refresh flag and specific tables ->>> run_pipeline --full-refresh table1,table2 +>>> run_pipeline --full-refresh table1 table2 • Full refreshed [table1, table2] >>> print_requests.py //pipelines @@ -57,7 +57,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Running pipeline with --full-refresh flag and --refresh flag ->>> run_pipeline --full-refresh table1,table2 --refresh table3,table4 +>>> run_pipeline --full-refresh table:table1 table:table2 --refresh table:table3 table:table4 • Refreshed [table3, table4] • Full refreshed [table1, table2] @@ -78,8 +78,29 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } } +=== Refreshing tables and fully refreshing a source file together +>>> run_pipeline --refresh table:table3 table:table4 --full-refresh file:pipeline_file.py +• Refreshed [table3, table4] +• Full refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "full_refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py" + ], + "refresh_selection": [ + "table3", + "table4" + ] + } +} + === Refreshing tables defined in a source file ->>> run_pipeline --refresh pipeline_file.py +>>> run_pipeline --refresh file:pipeline_file.py • Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] >>> print_requests.py //pipelines @@ -95,7 +116,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Fully refreshing tables defined in a source file ->>> run_pipeline --full-refresh pipeline_file.py +>>> run_pipeline --full-refresh file:pipeline_file.py • Full refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] >>> print_requests.py //pipelines @@ -111,7 +132,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Refreshing a source notebook ->>> run_pipeline --refresh src/pipeline_notebook.sql +>>> run_pipeline --refresh file:src/pipeline_notebook.sql • Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/src/pipeline_notebook] >>> print_requests.py //pipelines @@ -127,7 +148,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Refreshing an absolute workspace path ->>> run_pipeline --refresh /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py +>>> run_pipeline --refresh file:/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py • Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] >>> print_requests.py //pipelines @@ -143,7 +164,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Refreshing from a subdirectory ->>> withdir src run_pipeline --refresh ../pipeline_file.py +>>> withdir src run_pipeline --refresh file:../pipeline_file.py • Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] >>> print_requests.py //pipelines @@ -174,28 +195,132 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } } +=== Refreshing a source file and fully refreshing tables together +>>> run_pipeline --refresh file:pipeline_file.py --full-refresh table:table3 table:table4 +• Full refreshed [table3, table4] +• Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "full_refresh_selection": [ + "table3", + "table4" + ], + "refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py" + ] + } +} + === Rejecting missing sources and incompatible refresh modes ->>> musterr [CLI] pipelines run --refresh ./missing.sql --no-wait -Error: invalid --refresh: source file "./missing.sql" not found; use a table name or an existing source file +>>> musterr [CLI] pipelines run --refresh table: --restart --no-wait +Error: invalid --refresh: table: requires a table name + +>>> musterr [CLI] pipelines run --full-refresh file: --restart --no-wait +Error: invalid --full-refresh: file: requires a source file path + +>>> musterr [CLI] pipelines run --refresh file:./missing.sql --no-wait +Error: invalid --refresh: source file "./missing.sql" not found; use an existing source file ->>> musterr [CLI] pipelines run --refresh pipeline_file.py --full-refresh-all --no-wait +>>> musterr [CLI] pipelines run --refresh file:pipeline_file.py --full-refresh-all --no-wait Error: cannot use --refresh or --full-refresh together with --full-refresh-all, these flags are mutually exclusive ->>> musterr [CLI] pipelines run --full-refresh ./missing.sql --restart --no-wait -Error: invalid --full-refresh: source file "./missing.sql" not found; use a table name or an existing source file +>>> musterr [CLI] pipelines run --full-refresh file:./missing.sql --restart --no-wait +Error: invalid --full-refresh: source file "./missing.sql" not found; use an existing source file >>> print_requests.py //pipelines === Rejecting mixed selectors before restarting the pipeline ->>> musterr [CLI] pipelines run --refresh table1,pipeline_file.py --restart --no-wait +>>> musterr [CLI] pipelines run --refresh table:table1 file:pipeline_file.py --restart --no-wait Error: cannot mix table names and source files in --refresh; use only table names or only source files ->>> musterr [CLI] pipelines run --full-refresh table2,pipeline_file.py --restart --no-wait +>>> musterr [CLI] pipelines run --full-refresh table:table2 file:pipeline_file.py --restart --no-wait Error: cannot mix table names and source files in --full-refresh; use only table names or only source files >>> print_requests.py //pipelines ->>> musterr [CLI] bundle run my_pipeline --refresh table1,pipeline_file.py --restart --no-wait +>>> musterr [CLI] bundle run my_pipeline --refresh table:table1 file:pipeline_file.py --restart --no-wait Error: cannot mix table names and source files in --refresh; use only table names or only source files >>> print_requests.py //pipelines + +=== Repeated table selectors and backtick quoting +>>> run_pipeline --refresh table:table1 --refresh table2 +• Refreshed [table1, table2] + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "refresh_selection": [ + "table1", + "table2" + ] + } +} + +>>> run_pipeline --full-refresh table:`table1` `table2` +• Full refreshed [`table1`, `table2`] + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "full_refresh_selection": [ + "`table1`", + "`table2`" + ] + } +} + +=== Selecting multiple source files +>>> run_pipeline --refresh file:`pipeline_file.py` file:src/pipeline_notebook.sql +• Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py, /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/src/pipeline_notebook] + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py", + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/src/pipeline_notebook" + ] + } +} + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "full_refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py", + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/src/pipeline_notebook" + ] + } +} + +=== Rejecting invalid selector syntax before restart>>> musterr [CLI] pipelines run --refresh table1,table2 --restart --no-wait +Error: invalid argument "table1,table2" for "--refresh" flag: comma-separated selectors are no longer supported; separate selectors with spaces inside quotes, or repeat the flag; use backticks for literal commas +>>> musterr [CLI] pipelines run --full-refresh table1,table2 --restart --no-wait +Error: invalid argument "table1,table2" for "--full-refresh" flag: comma-separated selectors are no longer supported; separate selectors with spaces inside quotes, or repeat the flag; use backticks for literal commas +>>> musterr [CLI] pipelines run --refresh table:`table1 --restart --no-wait +Error: invalid argument "table:`table1" for "--refresh" flag: unclosed backtick in refresh selector +>>> musterr [CLI] pipelines run --refresh --restart --no-wait +Error: invalid argument "" for "--refresh" flag: expected at least one table or file selector + +>>> musterr [CLI] pipelines run --refresh flow:ingest --restart --no-wait +Error: invalid --refresh: flow selectors are not supported yet; select tables or files + +>>> print_requests.py //pipelines diff --git a/acceptance/pipelines/run/refresh-flags/script b/acceptance/pipelines/run/refresh-flags/script index 2df84af88c7..e54639bf000 100644 --- a/acceptance/pipelines/run/refresh-flags/script +++ b/acceptance/pipelines/run/refresh-flags/script @@ -22,54 +22,85 @@ trace run_pipeline --full-refresh-all trace print_requests.py //pipelines title "Running pipeline with --refresh flag and specific tables" -trace run_pipeline --refresh table1,table2 +trace run_pipeline --refresh "table1 table2" trace print_requests.py //pipelines title "Running pipeline with --full-refresh flag and specific tables" -trace run_pipeline --full-refresh table1,table2 +trace run_pipeline --full-refresh "table1 table2" trace print_requests.py //pipelines title "Running pipeline with --full-refresh flag and --refresh flag" -trace run_pipeline --full-refresh table1,table2 --refresh table3,table4 +trace run_pipeline --full-refresh "table:table1 table:table2" --refresh "table:table3 table:table4" +trace print_requests.py //pipelines + +title "Refreshing tables and fully refreshing a source file together" +trace run_pipeline --refresh "table:table3 table:table4" --full-refresh file:pipeline_file.py trace print_requests.py //pipelines title "Refreshing tables defined in a source file" -trace run_pipeline --refresh pipeline_file.py +trace run_pipeline --refresh file:pipeline_file.py trace print_requests.py //pipelines title "Fully refreshing tables defined in a source file" -trace run_pipeline --full-refresh pipeline_file.py +trace run_pipeline --full-refresh file:pipeline_file.py trace print_requests.py //pipelines title "Refreshing a source notebook" -trace run_pipeline --refresh src/pipeline_notebook.sql +trace run_pipeline --refresh file:src/pipeline_notebook.sql trace print_requests.py //pipelines title "Refreshing an absolute workspace path" -trace run_pipeline --refresh "$remote/pipeline_file.py" +trace run_pipeline --refresh "file:$remote/pipeline_file.py" trace print_requests.py //pipelines title "Refreshing from a subdirectory" -trace withdir src run_pipeline --refresh ../pipeline_file.py +trace withdir src run_pipeline --refresh file:../pipeline_file.py trace print_requests.py //pipelines title "Bundle run supports both file flags" -trace $CLI bundle run my_pipeline --refresh pipeline_file.py --full-refresh src/pipeline_notebook.sql &> LOG.bundle +trace $CLI bundle run my_pipeline --refresh file:pipeline_file.py --full-refresh file:src/pipeline_notebook.sql &> LOG.bundle +trace print_requests.py //pipelines + +title "Refreshing a source file and fully refreshing tables together" +trace run_pipeline --refresh file:pipeline_file.py --full-refresh "table:table3 table:table4" trace print_requests.py //pipelines title "Rejecting missing sources and incompatible refresh modes" -trace musterr $CLI pipelines run --refresh ./missing.sql --no-wait -trace musterr $CLI pipelines run --refresh pipeline_file.py --full-refresh-all --no-wait -trace musterr $CLI pipelines run --full-refresh ./missing.sql --restart --no-wait +trace musterr $CLI pipelines run --refresh table: --restart --no-wait +trace musterr $CLI pipelines run --full-refresh file: --restart --no-wait +trace musterr $CLI pipelines run --refresh file:./missing.sql --no-wait +trace musterr $CLI pipelines run --refresh file:pipeline_file.py --full-refresh-all --no-wait +trace musterr $CLI pipelines run --full-refresh file:./missing.sql --restart --no-wait touch "$OUT_REQUESTS" trace print_requests.py //pipelines title "Rejecting mixed selectors before restarting the pipeline" -trace musterr $CLI pipelines run --refresh table1,pipeline_file.py --restart --no-wait -trace musterr $CLI pipelines run --full-refresh table2,pipeline_file.py --restart --no-wait +trace musterr $CLI pipelines run --refresh "table:table1 file:pipeline_file.py" --restart --no-wait +trace musterr $CLI pipelines run --full-refresh "table:table2 file:pipeline_file.py" --restart --no-wait +touch "$OUT_REQUESTS" +trace print_requests.py //pipelines + +trace musterr $CLI bundle run my_pipeline --refresh "table:table1 file:pipeline_file.py" --restart --no-wait touch "$OUT_REQUESTS" trace print_requests.py //pipelines -trace musterr $CLI bundle run my_pipeline --refresh table1,pipeline_file.py --restart --no-wait +title "Repeated table selectors and backtick quoting" +trace run_pipeline --refresh table:table1 --refresh table2 +trace print_requests.py //pipelines +trace run_pipeline --full-refresh 'table:`table1` `table2`' +trace print_requests.py //pipelines + +title "Selecting multiple source files" +trace run_pipeline --refresh 'file:`pipeline_file.py` file:src/pipeline_notebook.sql' +trace print_requests.py //pipelines +trace $CLI bundle run my_pipeline --full-refresh 'file:pipeline_file.py file:src/pipeline_notebook.sql' &> LOG.bundle.files +trace print_requests.py //pipelines + +title "Rejecting invalid selector syntax before restart" +trace musterr $CLI pipelines run --refresh table1,table2 --restart --no-wait 2>&1 | grep -E '^(>>> |Error:)' +trace musterr $CLI pipelines run --full-refresh table1,table2 --restart --no-wait 2>&1 | grep -E '^(>>> |Error:)' +trace musterr $CLI pipelines run --refresh 'table:`table1' --restart --no-wait 2>&1 | grep -E '^(>>> |Error:)' +trace musterr $CLI pipelines run --refresh '' --restart --no-wait 2>&1 | grep -E '^(>>> |Error:)' +trace musterr $CLI pipelines run --refresh flow:ingest --restart --no-wait touch "$OUT_REQUESTS" trace print_requests.py //pipelines diff --git a/bundle/run/pipeline_options.go b/bundle/run/pipeline_options.go index e6a2a6753f9..d186d06aa44 100644 --- a/bundle/run/pipeline_options.go +++ b/bundle/run/pipeline_options.go @@ -3,6 +3,7 @@ package run import ( "errors" "fmt" + "slices" "strings" "github.com/databricks/cli/bundle/config/resources" @@ -34,12 +35,16 @@ type PipelineOptions struct { func (o *PipelineOptions) Define(fs *flag.FlagSet) { fs.BoolVar(&o.RefreshAll, "refresh-all", false, "Perform a full graph update.") - fs.StringSliceVar(&o.Refresh, "refresh", nil, "List of tables or source files to update. Local paths are relative to the current directory.") - fs.BoolVar(&o.FullRefreshAll, "full-refresh-all", false, "Perform a full graph reset and recompute.") - fs.StringSliceVar(&o.FullRefresh, "full-refresh", nil, "List of tables or source files to reset and recompute. Local paths are relative to the current directory.") + o.DefineRefreshFlags(fs) fs.BoolVar(&o.ValidateOnly, "validate-only", false, "Perform an update to validate graph correctness.") } +func (o *PipelineOptions) DefineRefreshFlags(fs *flag.FlagSet) { + fs.Var((*refreshSelectionFlag)(&o.Refresh), "refresh", "Space-separated table: or file: selectors to update. Bare names select tables. Repeatable.") + fs.BoolVar(&o.FullRefreshAll, "full-refresh-all", false, "Perform a full graph reset and recompute.") + fs.Var((*refreshSelectionFlag)(&o.FullRefresh), "full-refresh", "Space-separated table: or file: selectors to reset and recompute. Bare names select tables. Repeatable.") +} + // Validate returns if the combination of options is valid. func (o *PipelineOptions) Validate(pipeline *resources.Pipeline) error { var mutuallyExclusiveFlags []string @@ -90,5 +95,18 @@ func (o *PipelineOptions) toPayload(pipeline *resources.Pipeline, pipelineID str // using the pipeline-level development property while it is being deprecated. Development: development, } + // Legacy pipelines accept literal names rather than SQL identifiers. + if pipeline.Schema == "" { + payload.RefreshSelection = unquoteRefreshTables(payload.RefreshSelection) + payload.FullRefreshSelection = unquoteRefreshTables(payload.FullRefreshSelection) + } return payload, nil } + +func unquoteRefreshTables(tables []string) []string { + tables = slices.Clone(tables) + for i, table := range tables { + tables[i] = unquoteRefreshSelector(table) + } + return tables +} diff --git a/bundle/run/pipeline_options_test.go b/bundle/run/pipeline_options_test.go index b2f7c0a3f81..232cad4f11a 100644 --- a/bundle/run/pipeline_options_test.go +++ b/bundle/run/pipeline_options_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "testing" + "github.com/databricks/cli/bundle/config/resources" flag "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -42,7 +43,7 @@ func TestPipelineOptionsPathSelectionsPayload(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - payload, err := tt.opts.toPayload(nil, "pipeline-id", false) + payload, err := tt.opts.toPayload(&resources.Pipeline{}, "pipeline-id", false) require.NoError(t, err) data, err := json.Marshal(payload) require.NoError(t, err) @@ -51,6 +52,29 @@ func TestPipelineOptionsPathSelectionsPayload(t *testing.T) { } } +func TestPipelineOptionsQuotedTablesPayload(t *testing.T) { + for _, schema := range []string{"", "default"} { + t.Run("schema="+schema, func(t *testing.T) { + fs, opts := setupPipelineOptions(t) + err := fs.Parse([]string{"--refresh", "table:`a`` b,c` orders", "--full-refresh", "`daily rollup`"}) + require.NoError(t, err) + require.NoError(t, opts.ResolveRefreshSelectors(t.Context(), nil)) + pipeline := &resources.Pipeline{} + pipeline.Schema = schema + payload, err := opts.toPayload(pipeline, "pipeline-id", false) + require.NoError(t, err) + if schema == "" { + assert.Equal(t, []string{"a` b,c", "orders"}, payload.RefreshSelection) + assert.Equal(t, []string{"daily rollup"}, payload.FullRefreshSelection) + } else { + assert.Equal(t, []string{"`a`` b,c`", "orders"}, payload.RefreshSelection) + assert.Equal(t, []string{"`daily rollup`"}, payload.FullRefreshSelection) + } + assert.Equal(t, []string{"`a`` b,c`", "orders"}, opts.Refresh) + }) + } +} + func TestPipelineOptionsRejectMixedSelectors(t *testing.T) { tests := []struct { name string @@ -96,7 +120,7 @@ func TestPipelineOptionsRefreshAll(t *testing.T) { func TestPipelineOptionsRefresh(t *testing.T) { fs, opts := setupPipelineOptions(t) - err := fs.Parse([]string{`--refresh=arg1,arg2,arg3`}) + err := fs.Parse([]string{`--refresh=arg1 arg2 arg3`}) require.NoError(t, err) assert.Equal(t, []string{"arg1", "arg2", "arg3"}, opts.Refresh) } @@ -110,7 +134,7 @@ func TestPipelineOptionsFullRefreshAll(t *testing.T) { func TestPipelineOptionsFullRefresh(t *testing.T) { fs, opts := setupPipelineOptions(t) - err := fs.Parse([]string{`--full-refresh=arg1,arg2,arg3`}) + err := fs.Parse([]string{`--full-refresh=arg1 arg2 arg3`}) require.NoError(t, err) assert.Equal(t, []string{"arg1", "arg2", "arg3"}, opts.FullRefresh) } @@ -125,9 +149,9 @@ func TestPipelineOptionsValidateOnly(t *testing.T) { func TestPipelineOptionsValidateSuccessWithSingleOption(t *testing.T) { args := []string{ `--refresh-all`, - `--refresh=arg1,arg2,arg3`, + `--refresh=arg1 arg2 arg3`, `--full-refresh-all`, - `--full-refresh=arg1,arg2,arg3`, + `--full-refresh=arg1 arg2 arg3`, `--validate-only`, } for _, arg := range args { @@ -141,7 +165,7 @@ func TestPipelineOptionsValidateSuccessWithSingleOption(t *testing.T) { func TestPipelineOptionsValidateSuccessWithRefreshAndFullRefresh(t *testing.T) { fs, opts := setupPipelineOptions(t) - err := fs.Parse([]string{`--refresh=arg1,arg2`, `--full-refresh=arg3,arg4`}) + err := fs.Parse([]string{`--refresh=arg1 arg2`, `--full-refresh=arg3 arg4`}) require.NoError(t, err) err = opts.Validate(nil) assert.NoError(t, err) diff --git a/bundle/run/pipeline_refresh.go b/bundle/run/pipeline_refresh.go index 997579f8e44..f67026d6af3 100644 --- a/bundle/run/pipeline_refresh.go +++ b/bundle/run/pipeline_refresh.go @@ -9,6 +9,7 @@ import ( "path" "path/filepath" "strings" + "unicode" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/dyn" @@ -16,6 +17,71 @@ import ( "github.com/databricks/cli/libs/sync" ) +type refreshSelectionFlag []string + +func (f *refreshSelectionFlag) String() string { + return strings.Join(*f, " ") +} + +func (f *refreshSelectionFlag) Type() string { + return "selectors" +} + +func (f *refreshSelectionFlag) Set(value string) error { + var selectors []string + start := -1 + quoted := false + for i, c := range value { + if !quoted && unicode.IsSpace(c) { + if start >= 0 { + selectors = append(selectors, value[start:i]) + start = -1 + } + continue + } + if start < 0 { + start = i + } + switch c { + case '`': + quoted = !quoted + case ',': + if !quoted { + return errors.New("comma-separated selectors are no longer supported; separate selectors with spaces inside quotes, or repeat the flag; use backticks for literal commas") + } + } + } + if quoted { + return errors.New("unclosed backtick in refresh selector") + } + if start >= 0 { + selectors = append(selectors, value[start:]) + } + if len(selectors) == 0 { + return errors.New("expected at least one table or file selector") + } + *f = append(*f, selectors...) + return nil +} + +func unquoteRefreshSelector(value string) string { + var result strings.Builder + quoted := false + for i := 0; i < len(value); i++ { + if value[i] != '`' { + result.WriteByte(value[i]) + continue + } + if quoted && i+1 < len(value) && value[i+1] == '`' { + result.WriteByte('`') + i++ + } else { + quoted = !quoted + } + } + return result.String() +} + // ResolveRefreshSelectors resolves source files to workspace paths. func (o *PipelineOptions) ResolveRefreshSelectors(ctx context.Context, b *bundle.Bundle) error { r := refreshSelectorResolver{bundle: b} @@ -40,27 +106,38 @@ type refreshSelectorResolver struct { func (r *refreshSelectorResolver) resolve(ctx context.Context, selectors []string) (tables, paths []string, err error) { b := r.bundle for _, selector := range selectors { - // /Workspace/ distinguishes remote paths from local paths like /Users/... on macOS. - if strings.HasPrefix(selector, "/Workspace/") { - paths = append(paths, selector) + if selector == "``" { + return nil, nil, errors.New("table selector requires a table name") + } + if table, ok := strings.CutPrefix(selector, "table:"); ok { + if unquoteRefreshSelector(table) == "" { + return nil, nil, errors.New("table: requires a table name") + } + tables = append(tables, table) continue } - - // Quoted identifiers remain table selectors even when they contain path separators. - if strings.Contains(selector, "`") { + file, ok := strings.CutPrefix(selector, "file:") + if !ok { + if strings.HasPrefix(selector, "flow:") { + return nil, nil, errors.New("flow selectors are not supported yet; select tables or files") + } tables = append(tables, selector) continue } - - info, statErr := os.Stat(selector) - // A bare selector is a table unless it identifies an existing local file. - looksLikeFile := strings.ContainsAny(selector, `/\`) - if !looksLikeFile && (statErr != nil || info.IsDir()) { - tables = append(tables, selector) + file = unquoteRefreshSelector(file) + if file == "" { + return nil, nil, errors.New("file: requires a source file path") + } + selector = file + // /Workspace/ distinguishes remote paths from local paths like /Users/... on macOS. + if strings.HasPrefix(selector, "/Workspace/") { + paths = append(paths, selector) continue } + + info, statErr := os.Stat(selector) if errors.Is(statErr, fs.ErrNotExist) { - return nil, nil, fmt.Errorf("source file %q not found; use a table name or an existing source file", selector) + return nil, nil, fmt.Errorf("source file %q not found; use an existing source file", selector) } if statErr != nil { return nil, nil, fmt.Errorf("cannot read refresh selector %q: %w", selector, statErr) diff --git a/bundle/run/pipeline_refresh_test.go b/bundle/run/pipeline_refresh_test.go index 27b9c098bd1..5644f4f5a5c 100644 --- a/bundle/run/pipeline_refresh_test.go +++ b/bundle/run/pipeline_refresh_test.go @@ -1,6 +1,7 @@ package run import ( + "fmt" "os" "path/filepath" "strings" @@ -15,6 +16,54 @@ import ( "github.com/stretchr/testify/require" ) +func TestRefreshSelectionFlag(t *testing.T) { + var manyTables, manyFiles []string + for i := range 50 { + manyTables = append(manyTables, fmt.Sprintf("table_%d", i)) + manyFiles = append(manyFiles, fmt.Sprintf("file:src/pipeline_%d.py", i)) + } + tests := []struct { + name string + values []string + want []string + err string + }{ + {name: "50 tables", values: []string{strings.Join(manyTables, " ")}, want: manyTables}, + {name: "50 files", values: []string{strings.Join(manyFiles, " ")}, want: manyFiles}, + {name: "single table", values: []string{"orders"}, want: []string{"orders"}}, + {name: "table list", values: []string{"orders payments"}, want: []string{"orders", "payments"}}, + {name: "repeated flag", values: []string{"orders payments", "table:customers"}, want: []string{"orders", "payments", "table:customers"}}, + {name: "file list", values: []string{"file:a.py file:b.py"}, want: []string{"file:a.py", "file:b.py"}}, + {name: "whitespace", values: []string{" orders\tpayments\ncustomers\u2003returns "}, want: []string{"orders", "payments", "customers", "returns"}}, + {name: "quoted table", values: []string{"`order details` payments"}, want: []string{"`order details`", "payments"}}, + {name: "qualified identifier", values: []string{"table:`catalog`.`schema`.`order details`"}, want: []string{"table:`catalog`.`schema`.`order details`"}}, + {name: "quoted comma", values: []string{"table:`a,b` orders"}, want: []string{"table:`a,b`", "orders"}}, + {name: "escaped backtick", values: []string{"table:`a`` b,c` orders"}, want: []string{"table:`a`` b,c`", "orders"}}, + {name: "quoted file", values: []string{"file:`./my pipeline.py` file:b.py"}, want: []string{"file:`./my pipeline.py`", "file:b.py"}}, + {name: "empty value", values: []string{""}, err: "expected at least one"}, + {name: "whitespace only", values: []string{" \t "}, err: "expected at least one"}, + {name: "old comma list", values: []string{"orders,payments"}, err: "comma-separated selectors are no longer supported"}, + {name: "comma after quoted name", values: []string{"`orders`,payments"}, err: "comma-separated selectors"}, + {name: "unclosed backtick", values: []string{"orders `payments"}, err: "unclosed backtick"}, + {name: "unclosed escaped backtick", values: []string{"file:`a``"}, err: "unclosed backtick"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var flag refreshSelectionFlag + for _, value := range tt.values { + err := flag.Set(value) + if tt.err != "" { + require.ErrorContains(t, err, tt.err) + assert.Empty(t, flag) + return + } + require.NoError(t, err) + } + assert.Equal(t, tt.want, []string(flag)) + }) + } +} + func TestResolveRefreshSelectors(t *testing.T) { longTable := strings.Repeat("c", 100) + "." + strings.Repeat("s", 100) + "." + strings.Repeat("t", 100) tests := []struct { @@ -32,39 +81,55 @@ func TestResolveRefreshSelectors(t *testing.T) { tables: []string{"table1", "schema.table2", "schema.sql", "catalog.schema.sql", "`schema`.`table/with/slashes`", "`schema`.`sql`"}, }, { - name: "quoted table with the same name as a file", - selectors: []string{"`src/query.sql`"}, - tables: []string{"`src/query.sql`"}, + name: "bare table with the same name as a file", + selectors: []string{"orders"}, + tables: []string{"orders"}, }, { name: "mixed table and file selectors", - selectors: []string{"table1", "src/query.sql", "src/module.py"}, + selectors: []string{"table:table1", "file:src/query.sql", "file:src/module.py"}, tables: []string{"table1"}, paths: []string{"/Workspace/Shared/project/files/src/query.sql", "/Workspace/Shared/project/files/src/module.py"}, }, { name: "notebook extension removed", - selectors: []string{"src/notebook.py", "src/jupyter.ipynb"}, + selectors: []string{"file:src/notebook.py", "file:src/jupyter.ipynb"}, paths: []string{"/Workspace/Shared/project/files/src/notebook", "/Workspace/Shared/project/files/src/jupyter"}, }, { name: "workspace path", - selectors: []string{"/Workspace/Shared/remote.sql"}, - paths: []string{"/Workspace/Shared/remote.sql"}, + selectors: []string{"file:/Workspace/Shared/remote+part.sql"}, + paths: []string{"/Workspace/Shared/remote+part.sql"}, + }, + { + name: "explicit table identifiers", + selectors: []string{"table:orders", "table:catalog.schema.orders", "table:`file:orders`", "table:file:orders"}, + tables: []string{"orders", "catalog.schema.orders", "`file:orders`", "file:orders"}, }, - {name: "missing relative file", selectors: []string{"src/missing.sql"}, err: `source file "src/missing.sql" not found`}, - {name: "explicit missing filename", selectors: []string{"./missing.sql"}, err: `source file "./missing.sql" not found`}, + {name: "empty quoted table", selectors: []string{"table:``"}, err: "table: requires a table name"}, + {name: "empty bare table", selectors: []string{"``"}, err: "table selector requires a table name"}, + {name: "empty quoted file", selectors: []string{"file:``"}, err: "file: requires a source file path"}, + {name: "empty table", selectors: []string{"table:"}, err: "table: requires a table name"}, + {name: "unsupported flow", selectors: []string{"flow:ingest"}, err: "flow selectors are not supported"}, + {name: "quoted workspace path", selectors: []string{"file:`/Workspace/Shared/my pipeline.py`"}, paths: []string{"/Workspace/Shared/my pipeline.py"}}, + {name: "escaped backtick in workspace path", selectors: []string{"file:`/Workspace/Shared/a``b.py`"}, paths: []string{"/Workspace/Shared/a`b.py"}}, + {name: "local file with spaces", selectors: []string{"file:`src/my query.sql`"}, paths: []string{"/Workspace/Shared/project/files/src/my query.sql"}}, + {name: "empty path", selectors: []string{"file:"}, err: "file: requires a source file path"}, + {name: "missing relative file", selectors: []string{"file:src/missing.sql"}, err: `source file "src/missing.sql" not found`}, + {name: "explicit missing filename", selectors: []string{"file:./missing.sql"}, err: `source file "./missing.sql" not found`}, {name: "table named like a directory", selectors: []string{"src"}, tables: []string{"src"}}, - {name: "explicit directory", selectors: []string{"./src"}, err: "is a directory"}, - {name: "excluded file", selectors: []string{"src/excluded.sql"}, err: "is excluded from bundle sync"}, - {name: "invalid notebook", selectors: []string{"src/invalid.ipynb"}, err: "invalid Jupyter notebook file"}, + {name: "explicit directory", selectors: []string{"file:./src"}, err: "is a directory"}, + {name: "excluded file", selectors: []string{"file:src/excluded.sql"}, err: "is excluded from bundle sync"}, + {name: "invalid notebook", selectors: []string{"file:src/invalid.ipynb"}, err: "invalid Jupyter notebook file"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "orders"), []byte("SELECT 1;\n"), 0o600)) require.NoError(t, os.MkdirAll(filepath.Join(root, "src"), 0o700)) for name, content := range map[string]string{ "query.sql": "SELECT 1;\n", + "my query.sql": "SELECT 1;\n", "module.py": "print('file')\n", "notebook.py": "# Databricks notebook source\nprint('notebook')\n", "jupyter.ipynb": `{"cells":[],"metadata":{},"nbformat":4,"nbformat_minor":5}`, @@ -110,7 +175,7 @@ func TestResolveRefreshSelectorsFromSubdirectory(t *testing.T) { b := refreshTestBundle(root) b.BundleRootPath = filepath.Join(root, "project") for _, selector := range []string{"../../shared.sql", filepath.Join(root, "shared.sql")} { - tables, paths, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), []string{selector}) + tables, paths, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), []string{"file:" + selector}) require.NoError(t, err) assert.Empty(t, tables) assert.Equal(t, []string{"/Workspace/Shared/project/files/shared.sql"}, paths) @@ -122,7 +187,7 @@ func TestResolveRefreshSelectorsOutsideSyncRoot(t *testing.T) { require.NoError(t, os.Mkdir(filepath.Join(root, "project"), 0o700)) require.NoError(t, os.WriteFile(filepath.Join(root, "outside.sql"), []byte("SELECT 1;\n"), 0o600)) t.Chdir(filepath.Join(root, "project")) - _, _, err := (&refreshSelectorResolver{bundle: refreshTestBundle(filepath.Join(root, "project"))}).resolve(t.Context(), []string{"../outside.sql"}) + _, _, err := (&refreshSelectorResolver{bundle: refreshTestBundle(filepath.Join(root, "project"))}).resolve(t.Context(), []string{"file:../outside.sql"}) require.ErrorContains(t, err, "is outside the bundle sync root") } @@ -150,7 +215,7 @@ func TestResolveRefreshSelectorsDeploymentMode(t *testing.T) { b := refreshTestBundle(root) b.Config.Workspace.SyncMode = tt.mode b.Config.Presets.SourceLinkedDeployment = tt.sourceLinked - tables, paths, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), []string{"query.sql"}) + tables, paths, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), []string{"file:query.sql"}) require.NoError(t, err) assert.Empty(t, tables) assert.Equal(t, []string{tt.want}, paths) @@ -168,7 +233,7 @@ func TestResolveRefreshSelectorsUnresolvedWorkspacePath(t *testing.T) { enabled := true b.Config.Presets.SourceLinkedDeployment = &enabled b.DeploymentBundle.StateDB.OpenWithData("state", dstate.Database{}) - _, _, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), []string{"query.sql"}) + _, _, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), []string{"file:query.sql"}) require.ErrorContains(t, err, "use its absolute /Workspace/ path") } @@ -189,7 +254,7 @@ func TestResolveRefreshSelectorsDeployedSnapshot(t *testing.T) { }, }, }) - opts := PipelineOptions{Refresh: []string{"query.sql"}, FullRefresh: []string{"full_query.sql"}} + opts := PipelineOptions{Refresh: []string{"file:query.sql"}, FullRefresh: []string{"file:full_query.sql"}} require.NoError(t, opts.ResolveRefreshSelectors(t.Context(), b)) assert.Empty(t, opts.Refresh) assert.Empty(t, opts.FullRefresh) diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index d27c3a12695..8630f15dc08 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -108,6 +108,14 @@ parameter names. If the specified job does not use job parameters and the job has a Python file task or a Python wheel task, the second example applies. +For pipelines, --refresh and --full-refresh accept space-separated table: +or file: selectors inside quotes, or repeated flags. Bare names select tables. +For example: --refresh "orders payments" or --refresh "file:./src/a.py file:./src/b.py". +Use backticks around names or paths containing spaces or commas, and double a +backtick to escape it. In Bash and zsh, put values containing backticks in single quotes. +Comma-separated lists are no longer supported. +Deploy local file changes before running. Each flag accepts either tables or files. + --------------------------------------------------------- You can also use the bundle run command to execute scripts / commands in the same diff --git a/cmd/pipelines/run.go b/cmd/pipelines/run.go index f86fedb3b70..2594204c24a 100644 --- a/cmd/pipelines/run.go +++ b/cmd/pipelines/run.go @@ -239,22 +239,21 @@ KEY is the unique name of the pipeline to run, as defined in its YAML file. If there is only one pipeline in the project, KEY is optional and the pipeline will be auto-selected. Refreshes all tables in the pipeline unless otherwise specified. -Use --refresh or --full-refresh with table names, local source files, or absolute -/Workspace/ paths. Local files use the bundle sync configuration and paths are -relative to the current directory. Deploy local changes before running. -Each flag accepts either table names or source files, but cannot mix both. -Existing local files take precedence over unquoted table names. Use ./ to require -a file, or quote table identifiers with backticks to select an ambiguous table.`, +Use --refresh or --full-refresh with space-separated table: or file: +selectors inside quotes, or repeat the flag. Bare names select tables. +For example: --refresh "orders payments" or --refresh "file:./src/a.py file:./src/b.py". +Use backticks around names or paths containing spaces or commas, and double a +backtick to escape it. In Bash and zsh, put values containing backticks in single quotes. +Comma-separated lists are no longer supported. +Paths can be local source files or absolute /Workspace/ paths. Local paths are relative to the current directory and use the +bundle sync configuration. Deploy local changes before running. +Each flag accepts either tables or paths, but cannot mix both.`, } - var refresh []string - var fullRefreshAll bool - var fullRefresh []string + var pipelineOptions run.PipelineOptions pipelineGroup := cmdgroup.NewFlagGroup("Pipeline Run") - pipelineGroup.FlagSet().StringSliceVar(&refresh, "refresh", nil, "List of tables or source files to run. Local paths are relative to the current directory.") - pipelineGroup.FlagSet().BoolVar(&fullRefreshAll, "full-refresh-all", false, "Perform a full graph reset and recompute.") - pipelineGroup.FlagSet().StringSliceVar(&fullRefresh, "full-refresh", nil, "List of tables or source files to reset and recompute. Local paths are relative to the current directory.") + pipelineOptions.DefineRefreshFlags(pipelineGroup.FlagSet()) wrappedCmd := cmdgroup.NewCommandWithGroupFlag(cmd) wrappedCmd.AddFlagGroup(pipelineGroup) @@ -269,7 +268,7 @@ a file, or quote table identifiers with backticks to select an ambiguous table.` cmd.SetContext(ctx) runOptions := run.Options{ - Pipeline: run.PipelineOptions{Refresh: refresh, FullRefresh: fullRefresh, FullRefreshAll: fullRefreshAll}, + Pipeline: pipelineOptions, NoWait: noWait, } var key string From ccec702bd40100071573aac5edf4a1ec7ce14774 Mon Sep 17 00:00:00 2001 From: "chenyue.ma" Date: Tue, 10 Nov 2026 17:29:55 +0000 Subject: [PATCH 6/9] Clarify refresh selector usage and reject extra arguments --- .nextchanges/cli/pipeline-refresh-by-file.md | 2 +- acceptance/bundle/help/bundle-run/output.txt | 25 ++++++++----- .../bundle/run/refresh-flags/output.txt | 36 +++++++++---------- bundle/run/pipeline_options.go | 4 +-- cmd/bundle/run.go | 21 +++++++---- cmd/pipelines/run.go | 25 ++++++++----- 6 files changed, 67 insertions(+), 46 deletions(-) diff --git a/.nextchanges/cli/pipeline-refresh-by-file.md b/.nextchanges/cli/pipeline-refresh-by-file.md index cd3decf3d16..d158878735b 100644 --- a/.nextchanges/cli/pipeline-refresh-by-file.md +++ b/.nextchanges/cli/pipeline-refresh-by-file.md @@ -1 +1 @@ -* Support `file:` selectors in `pipelines run` and pipeline `bundle run` with `--refresh` and `--full-refresh`. Both flags now accept space-separated selectors inside quotes or repeated flags; bare names select tables and `table:` is optional. Replace comma-separated lists such as `--refresh orders,payments` with `--refresh "orders payments"`. Use backticks for names or paths containing spaces or commas, and double backticks to escape a literal backtick. ([#7184](https://github.com/databricks/cli/pull/7184)) +* Support `file:` selectors in `pipelines run` and pipeline `bundle run` with `--refresh` and `--full-refresh`. Both flags now accept space-separated selectors inside quotes or repeated flags. Replace comma-separated lists such as `--refresh orders,payments` with `--refresh "table:orders table:payments"`. Use backticks for names or paths containing spaces or commas, and double backticks to escape a literal backtick. ([#7184](https://github.com/databricks/cli/pull/7184)) diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt index a5c846fa795..cf08afef1f8 100644 --- a/acceptance/bundle/help/bundle-run/output.txt +++ b/acceptance/bundle/help/bundle-run/output.txt @@ -19,13 +19,20 @@ parameter names. If the specified job does not use job parameters and the job has a Python file task or a Python wheel task, the second example applies. -For pipelines, --refresh and --full-refresh accept space-separated table: -or file: selectors inside quotes, or repeated flags. Bare names select tables. -For example: --refresh "orders payments" or --refresh "file:./src/a.py file:./src/b.py". -Use backticks around names or paths containing spaces or commas, and double a -backtick to escape it. In Bash and zsh, put values containing backticks in single quotes. -Comma-separated lists are no longer supported. -Deploy local file changes before running. Each flag accepts either tables or files. +For pipeline runs, --refresh and --full-refresh accept: + Tables: orders or table:orders + Source files: file:./src/pipeline.py (file: is required) + +Put multiple selectors in quotes, or repeat the flag: + --refresh "orders payments" + --refresh "file:./src/a.py file:./src/b.py" + --refresh table:orders --refresh table:payments + +For a path containing spaces: + --refresh 'file:`./src/my pipeline.py`' + +Each refresh mode accepts either tables or files. +Deploy local file changes before running. --------------------------------------------------------- @@ -62,9 +69,9 @@ Job Task Flags: --sql-params stringToString A map from keys to values for jobs with SQL tasks. (default []) Pipeline Flags: - --full-refresh selectors Space-separated table: or file: selectors to reset and recompute. Bare names select tables. Repeatable. + --full-refresh selectors Space-separated table: or file: selectors to reset and recompute. Quote lists or repeat the flag. --full-refresh-all Perform a full graph reset and recompute. - --refresh selectors Space-separated table: or file: selectors to update. Bare names select tables. Repeatable. + --refresh selectors Space-separated table: or file: selectors to update. Quote lists or repeat the flag. --refresh-all Perform a full graph update. --validate-only Perform an update to validate graph correctness. diff --git a/acceptance/bundle/run/refresh-flags/output.txt b/acceptance/bundle/run/refresh-flags/output.txt index 1832375e9e7..6362a826b6e 100644 --- a/acceptance/bundle/run/refresh-flags/output.txt +++ b/acceptance/bundle/run/refresh-flags/output.txt @@ -7,14 +7,14 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Running pipeline with --refresh flag and specific tables >>> [CLI] bundle run my_pipeline --refresh table1 table2 -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][1] -Update ID: [UUID] +Update ID: [UUID][1] >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID]/updates", + "path": "/api/2.0/pipelines/[UUID][0]/updates", "body": { "refresh_selection": [ "table1", @@ -25,14 +25,14 @@ Update ID: [UUID] === Running pipeline with --full-refresh-all flag >>> [CLI] bundle run my_pipeline --full-refresh-all -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][2] -Update ID: [UUID] +Update ID: [UUID][2] >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID]/updates", + "path": "/api/2.0/pipelines/[UUID][0]/updates", "body": { "full_refresh": true } @@ -40,14 +40,14 @@ Update ID: [UUID] === Running pipeline with --full-refresh flag and specific tables >>> [CLI] bundle run my_pipeline --full-refresh table1 table2 -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][3] -Update ID: [UUID] +Update ID: [UUID][3] >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID]/updates", + "path": "/api/2.0/pipelines/[UUID][0]/updates", "body": { "full_refresh_selection": [ "table1", @@ -58,14 +58,14 @@ Update ID: [UUID] === Running pipeline with --full-refresh flag and --refresh flag >>> [CLI] bundle run my_pipeline --full-refresh table1 table2 --refresh table3 table4 -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][4] -Update ID: [UUID] +Update ID: [UUID][4] >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID]/updates", + "path": "/api/2.0/pipelines/[UUID][0]/updates", "body": { "full_refresh_selection": [ "table1", @@ -80,14 +80,14 @@ Update ID: [UUID] === Repeated refresh flags >>> [CLI] bundle run my_pipeline --refresh table1 --refresh table:table2 -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][5] -Update ID: [UUID] +Update ID: [UUID][5] >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID]/updates", + "path": "/api/2.0/pipelines/[UUID][0]/updates", "body": { "refresh_selection": [ "table1", @@ -98,14 +98,14 @@ Update ID: [UUID] === Quoted table selectors >>> [CLI] bundle run my_pipeline --refresh table:`table1` `table2` -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][6] -Update ID: [UUID] +Update ID: [UUID][6] >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID]/updates", + "path": "/api/2.0/pipelines/[UUID][0]/updates", "body": { "refresh_selection": [ "table1", diff --git a/bundle/run/pipeline_options.go b/bundle/run/pipeline_options.go index d186d06aa44..9dc3d7bf0e2 100644 --- a/bundle/run/pipeline_options.go +++ b/bundle/run/pipeline_options.go @@ -40,9 +40,9 @@ func (o *PipelineOptions) Define(fs *flag.FlagSet) { } func (o *PipelineOptions) DefineRefreshFlags(fs *flag.FlagSet) { - fs.Var((*refreshSelectionFlag)(&o.Refresh), "refresh", "Space-separated table: or file: selectors to update. Bare names select tables. Repeatable.") + fs.Var((*refreshSelectionFlag)(&o.Refresh), "refresh", "Space-separated table: or file: selectors to update. Quote lists or repeat the flag.") fs.BoolVar(&o.FullRefreshAll, "full-refresh-all", false, "Perform a full graph reset and recompute.") - fs.Var((*refreshSelectionFlag)(&o.FullRefresh), "full-refresh", "Space-separated table: or file: selectors to reset and recompute. Bare names select tables. Repeatable.") + fs.Var((*refreshSelectionFlag)(&o.FullRefresh), "full-refresh", "Space-separated table: or file: selectors to reset and recompute. Quote lists or repeat the flag.") } // Validate returns if the combination of options is valid. diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 8630f15dc08..4a28f59c605 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -108,13 +108,20 @@ parameter names. If the specified job does not use job parameters and the job has a Python file task or a Python wheel task, the second example applies. -For pipelines, --refresh and --full-refresh accept space-separated table: -or file: selectors inside quotes, or repeated flags. Bare names select tables. -For example: --refresh "orders payments" or --refresh "file:./src/a.py file:./src/b.py". -Use backticks around names or paths containing spaces or commas, and double a -backtick to escape it. In Bash and zsh, put values containing backticks in single quotes. -Comma-separated lists are no longer supported. -Deploy local file changes before running. Each flag accepts either tables or files. +For pipeline runs, --refresh and --full-refresh accept: + Tables: orders or table:orders + Source files: file:./src/pipeline.py (file: is required) + +Put multiple selectors in quotes, or repeat the flag: + --refresh "orders payments" + --refresh "file:./src/a.py file:./src/b.py" + --refresh table:orders --refresh table:payments + +For a path containing spaces: + --refresh 'file:` + "`./src/my pipeline.py`" + `' + +Each refresh mode accepts either tables or files. +Deploy local file changes before running. --------------------------------------------------------- diff --git a/cmd/pipelines/run.go b/cmd/pipelines/run.go index 2594204c24a..9a408ff57b5 100644 --- a/cmd/pipelines/run.go +++ b/cmd/pipelines/run.go @@ -239,15 +239,22 @@ KEY is the unique name of the pipeline to run, as defined in its YAML file. If there is only one pipeline in the project, KEY is optional and the pipeline will be auto-selected. Refreshes all tables in the pipeline unless otherwise specified. -Use --refresh or --full-refresh with space-separated table: or file: -selectors inside quotes, or repeat the flag. Bare names select tables. -For example: --refresh "orders payments" or --refresh "file:./src/a.py file:./src/b.py". -Use backticks around names or paths containing spaces or commas, and double a -backtick to escape it. In Bash and zsh, put values containing backticks in single quotes. -Comma-separated lists are no longer supported. -Paths can be local source files or absolute /Workspace/ paths. Local paths are relative to the current directory and use the -bundle sync configuration. Deploy local changes before running. -Each flag accepts either tables or paths, but cannot mix both.`, +For pipeline runs, --refresh and --full-refresh accept: + Tables: orders or table:orders + Source files: file:./src/pipeline.py (file: is required) + +Put multiple selectors in quotes, or repeat the flag: + --refresh "orders payments" + --refresh "file:./src/a.py file:./src/b.py" + --refresh table:orders --refresh table:payments + +For a path containing spaces: + --refresh 'file:` + "`./src/my pipeline.py`" + `' + +Each refresh mode accepts either tables or files. +Local file paths are relative to the current directory and map to deployed files. +Absolute /Workspace/ paths are also accepted. Deploy local changes before running.`, + Args: root.MaximumNArgs(1), } var pipelineOptions run.PipelineOptions From f815cff27a3a7f276a516de3b5a027acfc93344b Mon Sep 17 00:00:00 2001 From: "chenyue.ma" Date: Wed, 11 Nov 2026 06:14:09 +0000 Subject: [PATCH 7/9] Fix refresh validation and publishing mode handling --- .../bundle/run/refresh-flags/output.txt | 4 +- bundle/run/pipeline.go | 29 +++++ bundle/run/pipeline_options.go | 5 - bundle/run/pipeline_options_test.go | 23 ---- bundle/run/pipeline_refresh.go | 12 ++ bundle/run/pipeline_refresh_test.go | 14 +++ bundle/run/pipeline_test.go | 112 ++++++++++++++++++ 7 files changed, 169 insertions(+), 30 deletions(-) diff --git a/acceptance/bundle/run/refresh-flags/output.txt b/acceptance/bundle/run/refresh-flags/output.txt index 6362a826b6e..c39b8c6d7bd 100644 --- a/acceptance/bundle/run/refresh-flags/output.txt +++ b/acceptance/bundle/run/refresh-flags/output.txt @@ -108,8 +108,8 @@ Update ID: [UUID][6] "path": "/api/2.0/pipelines/[UUID][0]/updates", "body": { "refresh_selection": [ - "table1", - "table2" + "`table1`", + "`table2`" ] } } diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go index 97b8144d28b..19c2758efee 100644 --- a/bundle/run/pipeline.go +++ b/bundle/run/pipeline.go @@ -102,6 +102,24 @@ func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (output.RunOutp if err != nil { return nil, err } + var hasQuotedTables bool + for _, tables := range [][]string{req.RefreshSelection, req.FullRefreshSelection} { + for _, table := range tables { + hasQuotedTables = hasQuotedTables || strings.ContainsRune(table, '`') + } + } + if hasQuotedTables { + pipeline, err := w.Pipelines.GetByPipelineId(ctx, pipelineID) + if err != nil { + return nil, err + } + // Unset modes follow the backend's schema-based legacy detection. + if pipeline.EffectivePublishingMode == pipelines.PublishingModeLegacyPublishingMode || + (pipeline.EffectivePublishingMode == "" && pipeline.Spec.Schema == "") { + req.RefreshSelection = unquoteRefreshTables(req.RefreshSelection) + req.FullRefreshSelection = unquoteRefreshTables(req.FullRefreshSelection) + } + } res, err := w.Pipelines.StartUpdate(ctx, *req) if err != nil { @@ -185,6 +203,17 @@ func (r *pipelineRunner) Cancel(ctx context.Context) error { } func (r *pipelineRunner) Restart(ctx context.Context, opts *Options) (output.RunOutput, error) { + hasFiles := len(opts.Pipeline.RefreshByPath) > 0 || len(opts.Pipeline.FullRefreshByPath) > 0 + if hasFiles && !slices.Contains(opts.Pipeline.Refresh, "*") { + pipeline, err := r.bundle.WorkspaceClient(ctx).Pipelines.GetByPipelineId(ctx, r.pipeline.ID) + if err != nil { + return nil, err + } + if pipeline.Spec.Continuous { //nolint:staticcheck // Direct pipeline runs honor this setting. + return nil, errors.New("file selectors are not supported for continuous pipelines; use a triggered pipeline") + } + } + sp := cmdio.NewSpinner(ctx) sp.Update("Cancelling the active pipeline update") err := r.Cancel(ctx) diff --git a/bundle/run/pipeline_options.go b/bundle/run/pipeline_options.go index 9dc3d7bf0e2..2bf18c264ed 100644 --- a/bundle/run/pipeline_options.go +++ b/bundle/run/pipeline_options.go @@ -95,11 +95,6 @@ func (o *PipelineOptions) toPayload(pipeline *resources.Pipeline, pipelineID str // using the pipeline-level development property while it is being deprecated. Development: development, } - // Legacy pipelines accept literal names rather than SQL identifiers. - if pipeline.Schema == "" { - payload.RefreshSelection = unquoteRefreshTables(payload.RefreshSelection) - payload.FullRefreshSelection = unquoteRefreshTables(payload.FullRefreshSelection) - } return payload, nil } diff --git a/bundle/run/pipeline_options_test.go b/bundle/run/pipeline_options_test.go index 232cad4f11a..164f731f0ba 100644 --- a/bundle/run/pipeline_options_test.go +++ b/bundle/run/pipeline_options_test.go @@ -52,29 +52,6 @@ func TestPipelineOptionsPathSelectionsPayload(t *testing.T) { } } -func TestPipelineOptionsQuotedTablesPayload(t *testing.T) { - for _, schema := range []string{"", "default"} { - t.Run("schema="+schema, func(t *testing.T) { - fs, opts := setupPipelineOptions(t) - err := fs.Parse([]string{"--refresh", "table:`a`` b,c` orders", "--full-refresh", "`daily rollup`"}) - require.NoError(t, err) - require.NoError(t, opts.ResolveRefreshSelectors(t.Context(), nil)) - pipeline := &resources.Pipeline{} - pipeline.Schema = schema - payload, err := opts.toPayload(pipeline, "pipeline-id", false) - require.NoError(t, err) - if schema == "" { - assert.Equal(t, []string{"a` b,c", "orders"}, payload.RefreshSelection) - assert.Equal(t, []string{"daily rollup"}, payload.FullRefreshSelection) - } else { - assert.Equal(t, []string{"`a`` b,c`", "orders"}, payload.RefreshSelection) - assert.Equal(t, []string{"`daily rollup`"}, payload.FullRefreshSelection) - } - assert.Equal(t, []string{"`a`` b,c`", "orders"}, opts.Refresh) - }) - } -} - func TestPipelineOptionsRejectMixedSelectors(t *testing.T) { tests := []struct { name string diff --git a/bundle/run/pipeline_refresh.go b/bundle/run/pipeline_refresh.go index f67026d6af3..a4e7e544938 100644 --- a/bundle/run/pipeline_refresh.go +++ b/bundle/run/pipeline_refresh.go @@ -157,6 +157,18 @@ func (r *refreshSelectorResolver) resolve(ctx context.Context, selectors []strin if !filepath.IsLocal(rel) { return nil, nil, fmt.Errorf("source file %q is outside the bundle sync root", selector) } + if b.IsSourceLinkedDeployment() { + remoteName := filepath.ToSlash(rel) + isNotebook, _, err := notebook.DetectWithFS(b.SyncRoot, remoteName) + if err != nil { + return nil, nil, fmt.Errorf("cannot identify source file %q: %w", selector, err) + } + if isNotebook { + remoteName = notebook.StripExtension(remoteName) + } + paths = append(paths, path.Join(b.RemoteFilePath(), remoteName)) + continue + } if r.remoteNames == nil { // Refresh uses the deployed snapshot. if b.IsImmutableFolder() { diff --git a/bundle/run/pipeline_refresh_test.go b/bundle/run/pipeline_refresh_test.go index 5644f4f5a5c..5acda48d42a 100644 --- a/bundle/run/pipeline_refresh_test.go +++ b/bundle/run/pipeline_refresh_test.go @@ -261,3 +261,17 @@ func TestResolveRefreshSelectorsDeployedSnapshot(t *testing.T) { assert.Equal(t, []string{"/Workspace/Shared/snapshots/deployed/snapshot/files/query.sql"}, opts.RefreshByPath) assert.Equal(t, []string{"/Workspace/Shared/snapshots/deployed/snapshot/files/full_query.sql"}, opts.FullRefreshByPath) } + +func TestResolveRefreshSelectorsInPlaceExcludedFiles(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "query.sql"), []byte("SELECT 1;\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "notebook.py"), []byte("# Databricks notebook source\nprint('notebook')\n"), 0o600)) + t.Chdir(root) + b := refreshTestBundle(root) + b.Config.Workspace.SyncMode = config.SyncModeInPlace + b.Config.Sync.Exclude = []string{"query.sql", "notebook.py"} + tables, paths, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), []string{"file:query.sql", "file:notebook.py"}) + require.NoError(t, err) + assert.Empty(t, tables) + assert.Equal(t, []string{filepath.ToSlash(filepath.Join(root, "query.sql")), filepath.ToSlash(filepath.Join(root, "notebook"))}, paths) +} diff --git a/bundle/run/pipeline_test.go b/bundle/run/pipeline_test.go index 354216ee490..dee0f3be073 100644 --- a/bundle/run/pipeline_test.go +++ b/bundle/run/pipeline_test.go @@ -1,6 +1,7 @@ package run import ( + "errors" "testing" "time" @@ -111,3 +112,114 @@ func TestPipelineRunnerRestart(t *testing.T) { _, err := runner.Restart(ctx, &Options{}) require.NoError(t, err) } + +func TestPipelineRunnerQuotedTables(t *testing.T) { + tests := []struct { + name string + mode pipelines.PublishingMode + schema string + legacy bool + }{ + {name: "DPM with schema", mode: pipelines.PublishingModeDefaultPublishingMode, schema: "default"}, + {name: "DPM with target", mode: pipelines.PublishingModeDefaultPublishingMode}, + {name: "legacy", mode: pipelines.PublishingModeLegacyPublishingMode, legacy: true}, + {name: "unset mode with schema", schema: "default"}, + {name: "unset mode with target", legacy: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fs, opts := setupPipelineOptions(t) + require.NoError(t, fs.Parse([]string{"--refresh", "table:`a`` b,c` orders", "--full-refresh", "`daily rollup`"})) + require.NoError(t, opts.ResolveRefreshSelectors(t.Context(), nil)) + pipeline := &resources.Pipeline{ID: "123"} + pipeline.Schema = tt.schema + b := &bundle.Bundle{} + m := mocks.NewMockWorkspaceClient(t) + m.WorkspaceClient.Config = &sdk_config.Config{Host: "https://test.test"} + b.SetWorkpaceClient(m.WorkspaceClient) + spec := &pipelines.PipelineSpec{Schema: tt.schema} + if tt.schema == "" { + spec.Target = "default" + } + api := m.GetMockPipelinesAPI() + api.EXPECT().GetByPipelineId(mock.Anything, "123").Return(&pipelines.GetPipelineResponse{ + Spec: spec, EffectivePublishingMode: tt.mode, + }, nil).Once() + want := pipelines.StartUpdate{ + PipelineId: "123", RefreshSelection: []string{"`a`` b,c`", "orders"}, FullRefreshSelection: []string{"`daily rollup`"}, + } + if tt.legacy { + want.RefreshSelection = []string{"a` b,c", "orders"} + want.FullRefreshSelection = []string{"daily rollup"} + } + api.EXPECT().StartUpdate(mock.Anything, want).Return(&pipelines.StartUpdateResponse{UpdateId: "456"}, nil).Once() + runner := pipelineRunner{key: "test", bundle: b, pipeline: pipeline} + _, err := runner.Run(cmdio.MockDiscard(t.Context()), &Options{Pipeline: *opts, NoWait: true}) + require.NoError(t, err) + require.Equal(t, []string{"`a`` b,c`", "orders"}, opts.Refresh) + }) + } +} + +func TestPipelineRunnerRestartFileSelections(t *testing.T) { + tests := []struct { + name string + continuous bool + full bool + wildcard bool + getErr error + err string + }{ + {name: "continuous refresh", continuous: true, err: "file selectors are not supported for continuous pipelines"}, + {name: "continuous full refresh", continuous: true, full: true, err: "file selectors are not supported for continuous pipelines"}, + {name: "triggered refresh"}, + {name: "triggered full refresh", full: true}, + {name: "wildcard rollout is checked by backend", continuous: true, full: true, wildcard: true}, + {name: "cannot read deployed pipeline", getErr: errors.New("cannot read pipeline"), err: "cannot read pipeline"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pipeline := &resources.Pipeline{ID: "123"} + b := &bundle.Bundle{} + m := mocks.NewMockWorkspaceClient(t) + m.WorkspaceClient.Config = &sdk_config.Config{Host: "https://test.test"} + b.SetWorkpaceClient(m.WorkspaceClient) + api := m.GetMockPipelinesAPI() + if !tt.wildcard { + api.EXPECT().GetByPipelineId(mock.Anything, "123").Return(&pipelines.GetPipelineResponse{ + Spec: &pipelines.PipelineSpec{Continuous: tt.continuous}, //nolint:staticcheck // Exercise the supported legacy pipeline mode. + }, tt.getErr).Once() + } + opts := &Options{NoWait: true} + if tt.full { + opts.Pipeline.FullRefreshByPath = []string{"/Workspace/Shared/orders.py"} + } else { + opts.Pipeline.RefreshByPath = []string{"/Workspace/Shared/orders.py"} + } + if tt.wildcard { + opts.Pipeline.Refresh = []string{"*"} + } + if tt.err == "" { + wait := &pipelines.WaitGetPipelineIdle[struct{}]{ + Poll: func(time.Duration, func(*pipelines.GetPipelineResponse)) (*pipelines.GetPipelineResponse, error) { + return nil, nil + }, + } + api.EXPECT().Stop(mock.Anything, pipelines.StopRequest{PipelineId: "123"}).Return(wait, nil).Once() + api.EXPECT().StartUpdate(mock.Anything, pipelines.StartUpdate{ + PipelineId: "123", RefreshSelection: opts.Pipeline.Refresh, + RefreshSelectionByPath: opts.Pipeline.RefreshByPath, FullRefreshSelectionByPath: opts.Pipeline.FullRefreshByPath, + }).Return(&pipelines.StartUpdateResponse{UpdateId: "456"}, nil).Once() + } + runner := pipelineRunner{key: "test", bundle: b, pipeline: pipeline} + _, err := runner.Restart(cmdio.MockDiscard(t.Context()), opts) + if tt.err != "" { + require.ErrorContains(t, err, tt.err) + api.AssertNotCalled(t, "Stop", mock.Anything, mock.Anything) + api.AssertNotCalled(t, "StartUpdate", mock.Anything, mock.Anything) + } else { + require.NoError(t, err) + } + }) + } +} From c27cef9d689d6d4833853178af862abfb0d5ca0f Mon Sep 17 00:00:00 2001 From: "chenyue.ma" Date: Wed, 11 Nov 2026 19:48:56 +0000 Subject: [PATCH 8/9] Address pipeline refresh review feedback --- .nextchanges/cli/pipeline-refresh-by-file.md | 1 - .../pipeline-refresh-by-file.md | 1 + acceptance/bundle/help/bundle-run/output.txt | 5 +++- .../bundle/run/refresh-flags/output.txt | 9 ++++++- .../pipelines/run/refresh-flags/output.txt | 19 +++++++++++++- bundle/bundle.go | 3 ++- bundle/deploy/metadata/compute.go | 3 +-- bundle/run/pipeline.go | 6 +++-- bundle/run/pipeline_options.go | 25 +++++++++++++++++-- bundle/run/pipeline_refresh.go | 2 +- bundle/run/pipeline_test.go | 15 ++++++----- cmd/bundle/run.go | 15 +---------- cmd/pipelines/run.go | 16 +----------- 13 files changed, 71 insertions(+), 49 deletions(-) delete mode 100644 .nextchanges/cli/pipeline-refresh-by-file.md create mode 100644 .nextchanges/notable-changes/pipeline-refresh-by-file.md diff --git a/.nextchanges/cli/pipeline-refresh-by-file.md b/.nextchanges/cli/pipeline-refresh-by-file.md deleted file mode 100644 index d158878735b..00000000000 --- a/.nextchanges/cli/pipeline-refresh-by-file.md +++ /dev/null @@ -1 +0,0 @@ -* Support `file:` selectors in `pipelines run` and pipeline `bundle run` with `--refresh` and `--full-refresh`. Both flags now accept space-separated selectors inside quotes or repeated flags. Replace comma-separated lists such as `--refresh orders,payments` with `--refresh "table:orders table:payments"`. Use backticks for names or paths containing spaces or commas, and double backticks to escape a literal backtick. ([#7184](https://github.com/databricks/cli/pull/7184)) diff --git a/.nextchanges/notable-changes/pipeline-refresh-by-file.md b/.nextchanges/notable-changes/pipeline-refresh-by-file.md new file mode 100644 index 00000000000..fcf1c2083c5 --- /dev/null +++ b/.nextchanges/notable-changes/pipeline-refresh-by-file.md @@ -0,0 +1 @@ +* Add `file:` selectors to `pipelines run` and pipeline `bundle run`. `--refresh` and `--full-refresh` now use space-separated lists, such as `--refresh "orders payments"`, instead of comma-separated lists. Single names containing spaces now require backticks, e.g. ``--refresh 'table:`order details`'``. ([#7184](https://github.com/databricks/cli/pull/7184)) diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt index cf08afef1f8..b1f23d1bbc4 100644 --- a/acceptance/bundle/help/bundle-run/output.txt +++ b/acceptance/bundle/help/bundle-run/output.txt @@ -32,7 +32,10 @@ For a path containing spaces: --refresh 'file:`./src/my pipeline.py`' Each refresh mode accepts either tables or files. -Deploy local file changes before running. +Local file paths are relative to the current directory and map to deployed files. +Absolute /Workspace/ paths must match the deployed file path. +Deploy local changes before running. +With --restart, file selectors require a triggered pipeline. --------------------------------------------------------- diff --git a/acceptance/bundle/run/refresh-flags/output.txt b/acceptance/bundle/run/refresh-flags/output.txt index c39b8c6d7bd..83d62b3128c 100644 --- a/acceptance/bundle/run/refresh-flags/output.txt +++ b/acceptance/bundle/run/refresh-flags/output.txt @@ -6,6 +6,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Running pipeline with --refresh flag and specific tables + >>> [CLI] bundle run my_pipeline --refresh table1 table2 Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][1] @@ -24,6 +25,7 @@ Update ID: [UUID][1] } === Running pipeline with --full-refresh-all flag + >>> [CLI] bundle run my_pipeline --full-refresh-all Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][2] @@ -39,6 +41,7 @@ Update ID: [UUID][2] } === Running pipeline with --full-refresh flag and specific tables + >>> [CLI] bundle run my_pipeline --full-refresh table1 table2 Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][3] @@ -57,6 +60,7 @@ Update ID: [UUID][3] } === Running pipeline with --full-refresh flag and --refresh flag + >>> [CLI] bundle run my_pipeline --full-refresh table1 table2 --refresh table3 table4 Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][4] @@ -79,6 +83,7 @@ Update ID: [UUID][4] } === Repeated refresh flags + >>> [CLI] bundle run my_pipeline --refresh table1 --refresh table:table2 Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][5] @@ -97,6 +102,7 @@ Update ID: [UUID][5] } === Quoted table selectors + >>> [CLI] bundle run my_pipeline --refresh table:`table1` `table2` Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][6] @@ -114,7 +120,8 @@ Update ID: [UUID][6] } } -=== Invalid refresh syntax is rejected before restart>>> musterr [CLI] bundle run my_pipeline --refresh table1,table2 --restart --no-wait +=== Invalid refresh syntax is rejected before restart +>>> musterr [CLI] bundle run my_pipeline --refresh table1,table2 --restart --no-wait Error: invalid argument "table1,table2" for "--refresh" flag: comma-separated selectors are no longer supported; separate selectors with spaces inside quotes, or repeat the flag; use backticks for literal commas >>> musterr [CLI] bundle run my_pipeline --full-refresh table1,table2 --restart --no-wait Error: invalid argument "table1,table2" for "--full-refresh" flag: comma-separated selectors are no longer supported; separate selectors with spaces inside quotes, or repeat the flag; use backticks for literal commas diff --git a/acceptance/pipelines/run/refresh-flags/output.txt b/acceptance/pipelines/run/refresh-flags/output.txt index 6bdc78ef33e..f84c620f1e3 100644 --- a/acceptance/pipelines/run/refresh-flags/output.txt +++ b/acceptance/pipelines/run/refresh-flags/output.txt @@ -9,6 +9,7 @@ View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSP View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_refresh_[UNIQUE_NAME]?w=[WORKSPACE_ID] === Initialize all tables with --full-refresh-all + >>> run_pipeline --full-refresh-all • All tables are fully refreshed @@ -23,6 +24,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Running pipeline with --refresh flag and specific tables + >>> run_pipeline --refresh table1 table2 • Refreshed [table1, table2] @@ -40,6 +42,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Running pipeline with --full-refresh flag and specific tables + >>> run_pipeline --full-refresh table1 table2 • Full refreshed [table1, table2] @@ -57,6 +60,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Running pipeline with --full-refresh flag and --refresh flag + >>> run_pipeline --full-refresh table:table1 table:table2 --refresh table:table3 table:table4 • Refreshed [table3, table4] • Full refreshed [table1, table2] @@ -79,6 +83,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Refreshing tables and fully refreshing a source file together + >>> run_pipeline --refresh table:table3 table:table4 --full-refresh file:pipeline_file.py • Refreshed [table3, table4] • Full refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] @@ -100,6 +105,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Refreshing tables defined in a source file + >>> run_pipeline --refresh file:pipeline_file.py • Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] @@ -116,6 +122,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Fully refreshing tables defined in a source file + >>> run_pipeline --full-refresh file:pipeline_file.py • Full refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] @@ -132,6 +139,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Refreshing a source notebook + >>> run_pipeline --refresh file:src/pipeline_notebook.sql • Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/src/pipeline_notebook] @@ -148,6 +156,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Refreshing an absolute workspace path + >>> run_pipeline --refresh file:/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py • Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] @@ -164,6 +173,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Refreshing from a subdirectory + >>> withdir src run_pipeline --refresh file:../pipeline_file.py • Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] @@ -180,6 +190,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Bundle run supports both file flags + >>> print_requests.py //pipelines { "method": "POST", @@ -196,6 +207,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Refreshing a source file and fully refreshing tables together + >>> run_pipeline --refresh file:pipeline_file.py --full-refresh table:table3 table:table4 • Full refreshed [table3, table4] • Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] @@ -217,6 +229,7 @@ View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_r } === Rejecting missing sources and incompatible refresh modes + >>> musterr [CLI] pipelines run --refresh table: --restart --no-wait Error: invalid --refresh: table: requires a table name @@ -235,6 +248,7 @@ Error: invalid --full-refresh: source file "./missing.sql" not found; use an exi >>> print_requests.py //pipelines === Rejecting mixed selectors before restarting the pipeline + >>> musterr [CLI] pipelines run --refresh table:table1 file:pipeline_file.py --restart --no-wait Error: cannot mix table names and source files in --refresh; use only table names or only source files @@ -249,6 +263,7 @@ Error: cannot mix table names and source files in --refresh; use only table name >>> print_requests.py //pipelines === Repeated table selectors and backtick quoting + >>> run_pipeline --refresh table:table1 --refresh table2 • Refreshed [table1, table2] @@ -282,6 +297,7 @@ Error: cannot mix table names and source files in --refresh; use only table name } === Selecting multiple source files + >>> run_pipeline --refresh file:`pipeline_file.py` file:src/pipeline_notebook.sql • Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py, /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/src/pipeline_notebook] @@ -311,7 +327,8 @@ Error: cannot mix table names and source files in --refresh; use only table name } } -=== Rejecting invalid selector syntax before restart>>> musterr [CLI] pipelines run --refresh table1,table2 --restart --no-wait +=== Rejecting invalid selector syntax before restart +>>> musterr [CLI] pipelines run --refresh table1,table2 --restart --no-wait Error: invalid argument "table1,table2" for "--refresh" flag: comma-separated selectors are no longer supported; separate selectors with spaces inside quotes, or repeat the flag; use backticks for literal commas >>> musterr [CLI] pipelines run --full-refresh table1,table2 --restart --no-wait Error: invalid argument "table1,table2" for "--full-refresh" flag: comma-separated selectors are no longer supported; separate selectors with spaces inside quotes, or repeat the flag; use backticks for literal commas diff --git a/bundle/bundle.go b/bundle/bundle.go index ff22ce5d4a4..9f686a7e466 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -410,7 +410,8 @@ func (b *Bundle) GetSyncFiles(ctx context.Context) ([]fileset.File, error) { return list.Files(ctx) } -// RemoteFilePath returns the initialized sync destination. +// RemoteFilePath returns the workspace root for bundle files: +// the sync root for in-place deployments, otherwise workspace.file_path. func (b *Bundle) RemoteFilePath() string { if b.IsSourceLinkedDeployment() { return filepath.ToSlash(b.SyncRootPath) diff --git a/bundle/deploy/metadata/compute.go b/bundle/deploy/metadata/compute.go index 9cc260f7b91..86406fbd5d8 100644 --- a/bundle/deploy/metadata/compute.go +++ b/bundle/deploy/metadata/compute.go @@ -128,10 +128,9 @@ func (m *compute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics b.Metadata.Config.Resources.Dashboards = dashboardsMetadata // Set file upload destination of the bundle in metadata - b.Metadata.Config.Workspace.FilePath = b.Config.Workspace.FilePath + b.Metadata.Config.Workspace.FilePath = b.RemoteFilePath() // In source-linked deployment files are not copied and resources use source files, therefore we use sync path as file path in metadata if b.IsSourceLinkedDeployment() { - b.Metadata.Config.Workspace.FilePath = b.SyncRootPath b.Metadata.Config.Presets.SourceLinkedDeployment = true } diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go index 19c2758efee..ad0860c3356 100644 --- a/bundle/run/pipeline.go +++ b/bundle/run/pipeline.go @@ -113,6 +113,7 @@ func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (output.RunOutp if err != nil { return nil, err } + // Legacy mode takes raw names; default publishing mode parses SQL identifiers. // Unset modes follow the backend's schema-based legacy detection. if pipeline.EffectivePublishingMode == pipelines.PublishingModeLegacyPublishingMode || (pipeline.EffectivePublishingMode == "" && pipeline.Spec.Schema == "") { @@ -204,13 +205,14 @@ func (r *pipelineRunner) Cancel(ctx context.Context) error { func (r *pipelineRunner) Restart(ctx context.Context, opts *Options) (output.RunOutput, error) { hasFiles := len(opts.Pipeline.RefreshByPath) > 0 || len(opts.Pipeline.FullRefreshByPath) > 0 - if hasFiles && !slices.Contains(opts.Pipeline.Refresh, "*") { + // Check before stopping the active update. + if hasFiles { pipeline, err := r.bundle.WorkspaceClient(ctx).Pipelines.GetByPipelineId(ctx, r.pipeline.ID) if err != nil { return nil, err } if pipeline.Spec.Continuous { //nolint:staticcheck // Direct pipeline runs honor this setting. - return nil, errors.New("file selectors are not supported for continuous pipelines; use a triggered pipeline") + return nil, errors.New("file selectors with --restart require a triggered pipeline") } } diff --git a/bundle/run/pipeline_options.go b/bundle/run/pipeline_options.go index 2bf18c264ed..fd88f01d5a2 100644 --- a/bundle/run/pipeline_options.go +++ b/bundle/run/pipeline_options.go @@ -16,17 +16,19 @@ type PipelineOptions struct { // Perform a full graph update. RefreshAll bool - // List of tables or source files to update. + // Refresh holds CLI selectors before resolution and table names afterward. Refresh []string + // Workspace file paths resolved from Refresh. RefreshByPath []string // Perform a full graph reset and recompute. FullRefreshAll bool - // List of tables or source files to reset and recompute. + // FullRefresh holds CLI selectors before resolution and table names afterward. FullRefresh []string + // Workspace file paths resolved from FullRefresh. FullRefreshByPath []string // Perform an update to validate graph correctness. @@ -39,6 +41,25 @@ func (o *PipelineOptions) Define(fs *flag.FlagSet) { fs.BoolVar(&o.ValidateOnly, "validate-only", false, "Perform an update to validate graph correctness.") } +// PipelineRefreshHelp describes table and file refresh selectors. +const PipelineRefreshHelp = `For pipeline runs, --refresh and --full-refresh accept: + Tables: orders or table:orders + Source files: file:./src/pipeline.py (file: is required) + +Put multiple selectors in quotes, or repeat the flag: + --refresh "orders payments" + --refresh "file:./src/a.py file:./src/b.py" + --refresh table:orders --refresh table:payments + +For a path containing spaces: + --refresh 'file:` + "`./src/my pipeline.py`" + `' + +Each refresh mode accepts either tables or files. +Local file paths are relative to the current directory and map to deployed files. +Absolute /Workspace/ paths must match the deployed file path. +Deploy local changes before running. +With --restart, file selectors require a triggered pipeline.` + func (o *PipelineOptions) DefineRefreshFlags(fs *flag.FlagSet) { fs.Var((*refreshSelectionFlag)(&o.Refresh), "refresh", "Space-separated table: or file: selectors to update. Quote lists or repeat the flag.") fs.BoolVar(&o.FullRefreshAll, "full-refresh-all", false, "Perform a full graph reset and recompute.") diff --git a/bundle/run/pipeline_refresh.go b/bundle/run/pipeline_refresh.go index a4e7e544938..c1be406494e 100644 --- a/bundle/run/pipeline_refresh.go +++ b/bundle/run/pipeline_refresh.go @@ -82,7 +82,7 @@ func unquoteRefreshSelector(value string) string { return result.String() } -// ResolveRefreshSelectors resolves source files to workspace paths. +// ResolveRefreshSelectors separates table names and workspace file paths, then validates flag combinations. func (o *PipelineOptions) ResolveRefreshSelectors(ctx context.Context, b *bundle.Bundle) error { r := refreshSelectorResolver{bundle: b} var err error diff --git a/bundle/run/pipeline_test.go b/bundle/run/pipeline_test.go index dee0f3be073..0e0c84f5657 100644 --- a/bundle/run/pipeline_test.go +++ b/bundle/run/pipeline_test.go @@ -170,11 +170,12 @@ func TestPipelineRunnerRestartFileSelections(t *testing.T) { getErr error err string }{ - {name: "continuous refresh", continuous: true, err: "file selectors are not supported for continuous pipelines"}, - {name: "continuous full refresh", continuous: true, full: true, err: "file selectors are not supported for continuous pipelines"}, + {name: "continuous refresh", continuous: true, err: "file selectors with --restart require a triggered pipeline"}, + {name: "continuous full refresh", continuous: true, full: true, err: "file selectors with --restart require a triggered pipeline"}, {name: "triggered refresh"}, {name: "triggered full refresh", full: true}, - {name: "wildcard rollout is checked by backend", continuous: true, full: true, wildcard: true}, + {name: "continuous file selection with wildcard", continuous: true, full: true, wildcard: true, err: "file selectors with --restart require a triggered pipeline"}, + {name: "triggered file selection with wildcard", full: true, wildcard: true}, {name: "cannot read deployed pipeline", getErr: errors.New("cannot read pipeline"), err: "cannot read pipeline"}, } for _, tt := range tests { @@ -185,11 +186,9 @@ func TestPipelineRunnerRestartFileSelections(t *testing.T) { m.WorkspaceClient.Config = &sdk_config.Config{Host: "https://test.test"} b.SetWorkpaceClient(m.WorkspaceClient) api := m.GetMockPipelinesAPI() - if !tt.wildcard { - api.EXPECT().GetByPipelineId(mock.Anything, "123").Return(&pipelines.GetPipelineResponse{ - Spec: &pipelines.PipelineSpec{Continuous: tt.continuous}, //nolint:staticcheck // Exercise the supported legacy pipeline mode. - }, tt.getErr).Once() - } + api.EXPECT().GetByPipelineId(mock.Anything, "123").Return(&pipelines.GetPipelineResponse{ + Spec: &pipelines.PipelineSpec{Continuous: tt.continuous}, //nolint:staticcheck // Exercise the supported legacy pipeline mode. + }, tt.getErr).Once() opts := &Options{NoWait: true} if tt.full { opts.Pipeline.FullRefreshByPath = []string{"/Workspace/Shared/orders.py"} diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 4a28f59c605..618109f216f 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -108,20 +108,7 @@ parameter names. If the specified job does not use job parameters and the job has a Python file task or a Python wheel task, the second example applies. -For pipeline runs, --refresh and --full-refresh accept: - Tables: orders or table:orders - Source files: file:./src/pipeline.py (file: is required) - -Put multiple selectors in quotes, or repeat the flag: - --refresh "orders payments" - --refresh "file:./src/a.py file:./src/b.py" - --refresh table:orders --refresh table:payments - -For a path containing spaces: - --refresh 'file:` + "`./src/my pipeline.py`" + `' - -Each refresh mode accepts either tables or files. -Deploy local file changes before running. +` + run.PipelineRefreshHelp + ` --------------------------------------------------------- diff --git a/cmd/pipelines/run.go b/cmd/pipelines/run.go index 9a408ff57b5..cc85f32e6a0 100644 --- a/cmd/pipelines/run.go +++ b/cmd/pipelines/run.go @@ -239,21 +239,7 @@ KEY is the unique name of the pipeline to run, as defined in its YAML file. If there is only one pipeline in the project, KEY is optional and the pipeline will be auto-selected. Refreshes all tables in the pipeline unless otherwise specified. -For pipeline runs, --refresh and --full-refresh accept: - Tables: orders or table:orders - Source files: file:./src/pipeline.py (file: is required) - -Put multiple selectors in quotes, or repeat the flag: - --refresh "orders payments" - --refresh "file:./src/a.py file:./src/b.py" - --refresh table:orders --refresh table:payments - -For a path containing spaces: - --refresh 'file:` + "`./src/my pipeline.py`" + `' - -Each refresh mode accepts either tables or files. -Local file paths are relative to the current directory and map to deployed files. -Absolute /Workspace/ paths are also accepted. Deploy local changes before running.`, +` + run.PipelineRefreshHelp, Args: root.MaximumNArgs(1), } From 4e091936b2b0ed017d299d494591f093d46a13f5 Mon Sep 17 00:00:00 2001 From: "chenyue.ma" Date: Wed, 11 Nov 2026 20:30:04 +0000 Subject: [PATCH 9/9] Resolve in-place notebook selectors consistently --- acceptance/bundle/help/bundle-run/output.txt | 3 +- bundle/run/pipeline_options.go | 5 ++- bundle/run/pipeline_refresh.go | 35 ++++++++++++++++---- bundle/run/pipeline_refresh_test.go | 35 ++++++++++++++++++++ 4 files changed, 66 insertions(+), 12 deletions(-) diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt index b1f23d1bbc4..5e44a6839c6 100644 --- a/acceptance/bundle/help/bundle-run/output.txt +++ b/acceptance/bundle/help/bundle-run/output.txt @@ -33,9 +33,8 @@ For a path containing spaces: Each refresh mode accepts either tables or files. Local file paths are relative to the current directory and map to deployed files. -Absolute /Workspace/ paths must match the deployed file path. +Absolute /Workspace/ paths are also accepted. Deploy local changes before running. -With --restart, file selectors require a triggered pipeline. --------------------------------------------------------- diff --git a/bundle/run/pipeline_options.go b/bundle/run/pipeline_options.go index fd88f01d5a2..51f187741b4 100644 --- a/bundle/run/pipeline_options.go +++ b/bundle/run/pipeline_options.go @@ -56,9 +56,8 @@ For a path containing spaces: Each refresh mode accepts either tables or files. Local file paths are relative to the current directory and map to deployed files. -Absolute /Workspace/ paths must match the deployed file path. -Deploy local changes before running. -With --restart, file selectors require a triggered pipeline.` +Absolute /Workspace/ paths are also accepted. +Deploy local changes before running.` func (o *PipelineOptions) DefineRefreshFlags(fs *flag.FlagSet) { fs.Var((*refreshSelectionFlag)(&o.Refresh), "refresh", "Space-separated table: or file: selectors to update. Quote lists or repeat the flag.") diff --git a/bundle/run/pipeline_refresh.go b/bundle/run/pipeline_refresh.go index c1be406494e..af2e75a37af 100644 --- a/bundle/run/pipeline_refresh.go +++ b/bundle/run/pipeline_refresh.go @@ -131,6 +131,19 @@ func (r *refreshSelectorResolver) resolve(ctx context.Context, selectors []strin selector = file // /Workspace/ distinguishes remote paths from local paths like /Users/... on macOS. if strings.HasPrefix(selector, "/Workspace/") { + if b.IsSourceLinkedDeployment() { + rel, err := filepath.Rel(b.SyncRootPath, filepath.FromSlash(selector)) + if err == nil && filepath.IsLocal(rel) { + remotePath, err := r.resolveInPlacePath(selector, rel) + if err == nil { + paths = append(paths, remotePath) + continue + } + if !errors.Is(err, fs.ErrNotExist) { + return nil, nil, err + } + } + } paths = append(paths, selector) continue } @@ -158,15 +171,11 @@ func (r *refreshSelectorResolver) resolve(ctx context.Context, selectors []strin return nil, nil, fmt.Errorf("source file %q is outside the bundle sync root", selector) } if b.IsSourceLinkedDeployment() { - remoteName := filepath.ToSlash(rel) - isNotebook, _, err := notebook.DetectWithFS(b.SyncRoot, remoteName) + remotePath, err := r.resolveInPlacePath(selector, rel) if err != nil { - return nil, nil, fmt.Errorf("cannot identify source file %q: %w", selector, err) - } - if isNotebook { - remoteName = notebook.StripExtension(remoteName) + return nil, nil, err } - paths = append(paths, path.Join(b.RemoteFilePath(), remoteName)) + paths = append(paths, remotePath) continue } if r.remoteNames == nil { @@ -197,6 +206,18 @@ func (r *refreshSelectorResolver) resolve(ctx context.Context, selectors []strin return tables, paths, nil } +func (r *refreshSelectorResolver) resolveInPlacePath(selector, rel string) (string, error) { + remoteName := filepath.ToSlash(rel) + isNotebook, _, err := notebook.DetectWithFS(r.bundle.SyncRoot, remoteName) + if err != nil { + return "", fmt.Errorf("cannot identify source file %q: %w", selector, err) + } + if isNotebook { + remoteName = notebook.StripExtension(remoteName) + } + return path.Join(r.bundle.RemoteFilePath(), remoteName), nil +} + // refreshSourcePaths maps local files to their deployed names. func refreshSourcePaths(ctx context.Context, b *bundle.Bundle) (map[string]string, error) { files, err := b.GetSyncFiles(ctx) diff --git a/bundle/run/pipeline_refresh_test.go b/bundle/run/pipeline_refresh_test.go index 5acda48d42a..6dcefea335b 100644 --- a/bundle/run/pipeline_refresh_test.go +++ b/bundle/run/pipeline_refresh_test.go @@ -275,3 +275,38 @@ func TestResolveRefreshSelectorsInPlaceExcludedFiles(t *testing.T) { assert.Empty(t, tables) assert.Equal(t, []string{filepath.ToSlash(filepath.Join(root, "query.sql")), filepath.ToSlash(filepath.Join(root, "notebook"))}, paths) } + +func TestResolveRefreshSelectorsInPlaceWorkspacePaths(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "notebook.py"), []byte("# Databricks notebook source\nprint('notebook')\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "module.py"), []byte("print('file')\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "invalid.ipynb"), []byte(`{}`), 0o600)) + b := refreshTestBundle(root) + b.SyncRootPath = filepath.FromSlash("/Workspace/Users/test/project") + b.Config.Workspace.SyncMode = config.SyncModeInPlace + b.Config.Sync.Exclude = []string{"notebook.py"} + + for _, tt := range []struct { + name string + path string + want string + err string + }{ + {name: "source notebook", path: "/Workspace/Users/test/project/notebook.py", want: "/Workspace/Users/test/project/notebook"}, + {name: "regular file", path: "/Workspace/Users/test/project/module.py", want: "/Workspace/Users/test/project/module.py"}, + {name: "deployed notebook", path: "/Workspace/Users/test/project/notebook", want: "/Workspace/Users/test/project/notebook"}, + {name: "outside sync root", path: "/Workspace/Users/test/project-other/notebook.py", want: "/Workspace/Users/test/project-other/notebook.py"}, + {name: "invalid notebook", path: "/Workspace/Users/test/project/invalid.ipynb", err: "invalid Jupyter notebook file"}, + } { + t.Run(tt.name, func(t *testing.T) { + tables, paths, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), []string{"file:" + tt.path}) + if tt.err != "" { + require.ErrorContains(t, err, tt.err) + return + } + require.NoError(t, err) + assert.Empty(t, tables) + assert.Equal(t, []string{tt.want}, paths) + }) + } +}