diff --git a/.nextchanges/notable-changes/pipeline-refresh-by-file.md b/.nextchanges/notable-changes/pipeline-refresh-by-file.md new file mode 100644 index 00000000000..fcf1c2083c5 --- /dev/null +++ b/.nextchanges/notable-changes/pipeline-refresh-by-file.md @@ -0,0 +1 @@ +* Add `file:` selectors to `pipelines run` and pipeline `bundle run`. `--refresh` and `--full-refresh` now use space-separated lists, such as `--refresh "orders payments"`, instead of comma-separated lists. Single names containing spaces now require backticks, e.g. ``--refresh 'table:`order details`'``. ([#7184](https://github.com/databricks/cli/pull/7184)) diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt index a6b453c8834..5e44a6839c6 100644 --- a/acceptance/bundle/help/bundle-run/output.txt +++ b/acceptance/bundle/help/bundle-run/output.txt @@ -19,6 +19,23 @@ parameter names. If the specified job does not use job parameters and the job has a Python file task or a Python wheel task, the second example applies. +For pipeline runs, --refresh and --full-refresh accept: + Tables: orders or table:orders + Source files: file:./src/pipeline.py (file: is required) + +Put multiple selectors in quotes, or repeat the flag: + --refresh "orders payments" + --refresh "file:./src/a.py file:./src/b.py" + --refresh table:orders --refresh table:payments + +For a path containing spaces: + --refresh 'file:`./src/my pipeline.py`' + +Each refresh mode accepts either tables or files. +Local file paths are relative to the current directory and map to deployed files. +Absolute /Workspace/ paths are also accepted. +Deploy local changes before running. + --------------------------------------------------------- You can also use the bundle run command to execute scripts / commands in the same @@ -54,11 +71,11 @@ Job Task Flags: --sql-params stringToString A map from keys to values for jobs with SQL tasks. (default []) Pipeline Flags: - --full-refresh strings List of tables to reset and recompute. - --full-refresh-all Perform a full graph reset and recompute. - --refresh strings List of tables to update. - --refresh-all Perform a full graph update. - --validate-only Perform an update to validate graph correctness. + --full-refresh selectors Space-separated table: or file: selectors to reset and recompute. Quote lists or repeat the flag. + --full-refresh-all Perform a full graph reset and recompute. + --refresh selectors Space-separated table: or file: selectors to update. Quote lists or repeat the flag. + --refresh-all Perform a full graph update. + --validate-only Perform an update to validate graph correctness. Flags: -h, --help help for run diff --git a/acceptance/bundle/run/refresh-flags/output.txt b/acceptance/bundle/run/refresh-flags/output.txt index f5b077ea23d..83d62b3128c 100644 --- a/acceptance/bundle/run/refresh-flags/output.txt +++ b/acceptance/bundle/run/refresh-flags/output.txt @@ -7,7 +7,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Running pipeline with --refresh flag and specific tables ->>> [CLI] bundle run my_pipeline --refresh table1,table2 +>>> [CLI] bundle run my_pipeline --refresh table1 table2 Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][1] Update ID: [UUID][1] @@ -42,7 +42,7 @@ Update ID: [UUID][2] === Running pipeline with --full-refresh flag and specific tables ->>> [CLI] bundle run my_pipeline --full-refresh table1,table2 +>>> [CLI] bundle run my_pipeline --full-refresh table1 table2 Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][3] Update ID: [UUID][3] @@ -61,7 +61,7 @@ Update ID: [UUID][3] === Running pipeline with --full-refresh flag and --refresh flag ->>> [CLI] bundle run my_pipeline --full-refresh table1,table2 --refresh table3,table4 +>>> [CLI] bundle run my_pipeline --full-refresh table1 table2 --refresh table3 table4 Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][4] Update ID: [UUID][4] @@ -81,3 +81,53 @@ Update ID: [UUID][4] ] } } + +=== Repeated refresh flags + +>>> [CLI] bundle run my_pipeline --refresh table1 --refresh table:table2 +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][5] + +Update ID: [UUID][5] + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID][0]/updates", + "body": { + "refresh_selection": [ + "table1", + "table2" + ] + } +} + +=== Quoted table selectors + +>>> [CLI] bundle run my_pipeline --refresh table:`table1` `table2` +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][6] + +Update ID: [UUID][6] + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID][0]/updates", + "body": { + "refresh_selection": [ + "`table1`", + "`table2`" + ] + } +} + +=== Invalid refresh syntax is rejected before restart +>>> musterr [CLI] bundle run my_pipeline --refresh table1,table2 --restart --no-wait +Error: invalid argument "table1,table2" for "--refresh" flag: comma-separated selectors are no longer supported; separate selectors with spaces inside quotes, or repeat the flag; use backticks for literal commas +>>> musterr [CLI] bundle run my_pipeline --full-refresh table1,table2 --restart --no-wait +Error: invalid argument "table1,table2" for "--full-refresh" flag: comma-separated selectors are no longer supported; separate selectors with spaces inside quotes, or repeat the flag; use backticks for literal commas +>>> musterr [CLI] bundle run my_pipeline --refresh table:`table1 --restart --no-wait +Error: invalid argument "table:`table1" for "--refresh" flag: unclosed backtick in refresh selector +>>> musterr [CLI] bundle run my_pipeline --refresh --restart --no-wait +Error: invalid argument "" for "--refresh" flag: expected at least one table or file selector + +>>> print_requests.py //pipelines diff --git a/acceptance/bundle/run/refresh-flags/script b/acceptance/bundle/run/refresh-flags/script index 7c788c42239..eaa3ee2e9fe 100644 --- a/acceptance/bundle/run/refresh-flags/script +++ b/acceptance/bundle/run/refresh-flags/script @@ -2,7 +2,7 @@ trace $CLI bundle deploy rm "$OUT_REQUESTS" title "Running pipeline with --refresh flag and specific tables" -trace $CLI bundle run my_pipeline --refresh table1,table2 +trace $CLI bundle run my_pipeline --refresh "table1 table2" trace print_requests.py //pipelines title "Running pipeline with --full-refresh-all flag" @@ -10,9 +10,25 @@ trace $CLI bundle run my_pipeline --full-refresh-all trace print_requests.py //pipelines title "Running pipeline with --full-refresh flag and specific tables" -trace $CLI bundle run my_pipeline --full-refresh table1,table2 +trace $CLI bundle run my_pipeline --full-refresh "table1 table2" trace print_requests.py //pipelines title "Running pipeline with --full-refresh flag and --refresh flag" -trace $CLI bundle run my_pipeline --full-refresh table1,table2 --refresh table3,table4 +trace $CLI bundle run my_pipeline --full-refresh "table1 table2" --refresh "table3 table4" +trace print_requests.py //pipelines + +title "Repeated refresh flags" +trace $CLI bundle run my_pipeline --refresh table1 --refresh table:table2 +trace print_requests.py //pipelines + +title "Quoted table selectors" +trace $CLI bundle run my_pipeline --refresh 'table:`table1` `table2`' +trace print_requests.py //pipelines + +title "Invalid refresh syntax is rejected before restart" +trace musterr $CLI bundle run my_pipeline --refresh table1,table2 --restart --no-wait 2>&1 | grep -E '^(>>> |Error:)' +trace musterr $CLI bundle run my_pipeline --full-refresh table1,table2 --restart --no-wait 2>&1 | grep -E '^(>>> |Error:)' +trace musterr $CLI bundle run my_pipeline --refresh 'table:`table1' --restart --no-wait 2>&1 | grep -E '^(>>> |Error:)' +trace musterr $CLI bundle run my_pipeline --refresh '' --restart --no-wait 2>&1 | grep -E '^(>>> |Error:)' +touch "$OUT_REQUESTS" trace print_requests.py //pipelines diff --git a/acceptance/pipelines/run/refresh-flags/databricks.yml b/acceptance/pipelines/run/refresh-flags/databricks.yml index 37e89920ad4..7fe4b3f27b5 100644 --- a/acceptance/pipelines/run/refresh-flags/databricks.yml +++ b/acceptance/pipelines/run/refresh-flags/databricks.yml @@ -1,10 +1,29 @@ bundle: name: test-pipeline-run-flags +variables: + unique_name: + description: Unique name for test resources + +workspace: + root_path: ~/.bundle/${bundle.name}/${var.unique_name} + +presets: + pipelines_development: true + resources: + schemas: + test_schema: + name: pipeline_refresh_${var.unique_name} + catalog_name: main pipelines: my_pipeline: - name: test-pipeline-flags + name: test-pipeline-flags-${var.unique_name} + catalog: main + schema: ${resources.schemas.test_schema.name} + serverless: true libraries: - file: path: pipeline_file.py + - notebook: + path: src/pipeline_notebook.sql diff --git a/acceptance/pipelines/run/refresh-flags/out.test.toml b/acceptance/pipelines/run/refresh-flags/out.test.toml index 18b1a88417e..c7c6f58ed6e 100644 --- a/acceptance/pipelines/run/refresh-flags/out.test.toml +++ b/acceptance/pipelines/run/refresh-flags/out.test.toml @@ -1 +1 @@ -Cloud = false +Cloud = true diff --git a/acceptance/pipelines/run/refresh-flags/output.txt b/acceptance/pipelines/run/refresh-flags/output.txt index 791a73fa999..f84c620f1e3 100644 --- a/acceptance/pipelines/run/refresh-flags/output.txt +++ b/acceptance/pipelines/run/refresh-flags/output.txt @@ -1,120 +1,343 @@ >>> [CLI] pipelines deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files... Created pipelines.my_pipeline -Files: 2 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID] +Created schemas.test_schema +Files: 3 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] +View your schema test_schema here: [DATABRICKS_URL]/explore/data/main/pipeline_refresh_[UNIQUE_NAME]?w=[WORKSPACE_ID] + +=== Initialize all tables with --full-refresh-all + +>>> run_pipeline --full-refresh-all +• All tables are fully refreshed + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "full_refresh": true + } +} === Running pipeline with --refresh flag and specific tables ->>> [CLI] pipelines run --refresh table1,table2 -Recommendation: This command runs the last deployed version of the code +>>> run_pipeline --refresh table1 table2 +• Refreshed [table1, table2] + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "refresh_selection": [ + "table1", + "table2" + ] + } +} -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. +=== Running pipeline with --full-refresh flag and specific tables -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][1] +>>> run_pipeline --full-refresh table1 table2 +• Full refreshed [table1, table2] -Update ID: [UUID][1] +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "full_refresh_selection": [ + "table1", + "table2" + ] + } +} -Update for pipeline completed successfully. +=== Running pipeline with --full-refresh flag and --refresh flag -Pipeline configurations for this update: -• All tables are refreshed +>>> run_pipeline --full-refresh table:table1 table:table2 --refresh table:table3 table:table4 +• Refreshed [table3, table4] +• Full refreshed [table1, table2] >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID][0]/updates", + "path": "/api/2.0/pipelines/[UUID]/updates", "body": { - "refresh_selection": [ + "development": true, + "full_refresh_selection": [ "table1", "table2" + ], + "refresh_selection": [ + "table3", + "table4" ] } } -=== Running pipeline with --full-refresh-all flag +=== Refreshing tables and fully refreshing a source file together ->>> [CLI] pipelines run --full-refresh-all -Recommendation: This command runs the last deployed version of the code +>>> run_pipeline --refresh table:table3 table:table4 --full-refresh file:pipeline_file.py +• Refreshed [table3, table4] +• Full refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "full_refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py" + ], + "refresh_selection": [ + "table3", + "table4" + ] + } +} + +=== Refreshing tables defined in a source file -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][2] +>>> run_pipeline --refresh file:pipeline_file.py +• Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] -Update ID: [UUID][2] +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py" + ] + } +} -Update for pipeline completed successfully. +=== Fully refreshing tables defined in a source file -Pipeline configurations for this update: -• All tables are refreshed +>>> run_pipeline --full-refresh file:pipeline_file.py +• Full refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID][0]/updates", + "path": "/api/2.0/pipelines/[UUID]/updates", "body": { - "full_refresh": true + "development": true, + "full_refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py" + ] } } -=== Running pipeline with --full-refresh flag and specific tables +=== Refreshing a source notebook ->>> [CLI] pipelines run --full-refresh table1,table2 -Recommendation: This command runs the last deployed version of the code +>>> run_pipeline --refresh file:src/pipeline_notebook.sql +• Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/src/pipeline_notebook] -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/src/pipeline_notebook" + ] + } +} + +=== Refreshing an absolute workspace path -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][3] +>>> run_pipeline --refresh file:/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py +• Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] -Update ID: [UUID][3] +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py" + ] + } +} -Update for pipeline completed successfully. +=== Refreshing from a subdirectory -Pipeline configurations for this update: -• All tables are refreshed +>>> withdir src run_pipeline --refresh file:../pipeline_file.py +• Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID][0]/updates", + "path": "/api/2.0/pipelines/[UUID]/updates", "body": { + "development": true, + "refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py" + ] + } +} + +=== Bundle run supports both file flags + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "full_refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/src/pipeline_notebook" + ], + "refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py" + ] + } +} + +=== Refreshing a source file and fully refreshing tables together + +>>> run_pipeline --refresh file:pipeline_file.py --full-refresh table:table3 table:table4 +• Full refreshed [table3, table4] +• Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py] + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, "full_refresh_selection": [ - "table1", - "table2" + "table3", + "table4" + ], + "refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py" ] } } -=== Running pipeline with --full-refresh flag and --refresh flag +=== Rejecting missing sources and incompatible refresh modes + +>>> musterr [CLI] pipelines run --refresh table: --restart --no-wait +Error: invalid --refresh: table: requires a table name ->>> [CLI] pipelines run --full-refresh table1,table2 --refresh table3,table4 -Recommendation: This command runs the last deployed version of the code +>>> musterr [CLI] pipelines run --full-refresh file: --restart --no-wait +Error: invalid --full-refresh: file: requires a source file path -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. +>>> musterr [CLI] pipelines run --refresh file:./missing.sql --no-wait +Error: invalid --refresh: source file "./missing.sql" not found; use an existing source file -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][4] +>>> musterr [CLI] pipelines run --refresh file:pipeline_file.py --full-refresh-all --no-wait +Error: cannot use --refresh or --full-refresh together with --full-refresh-all, these flags are mutually exclusive -Update ID: [UUID][4] +>>> musterr [CLI] pipelines run --full-refresh file:./missing.sql --restart --no-wait +Error: invalid --full-refresh: source file "./missing.sql" not found; use an existing source file + +>>> print_requests.py //pipelines -Update for pipeline completed successfully. +=== Rejecting mixed selectors before restarting the pipeline -Pipeline configurations for this update: -• All tables are refreshed +>>> musterr [CLI] pipelines run --refresh table:table1 file:pipeline_file.py --restart --no-wait +Error: cannot mix table names and source files in --refresh; use only table names or only source files + +>>> musterr [CLI] pipelines run --full-refresh table:table2 file:pipeline_file.py --restart --no-wait +Error: cannot mix table names and source files in --full-refresh; use only table names or only source files + +>>> print_requests.py //pipelines + +>>> musterr [CLI] bundle run my_pipeline --refresh table:table1 file:pipeline_file.py --restart --no-wait +Error: cannot mix table names and source files in --refresh; use only table names or only source files + +>>> print_requests.py //pipelines + +=== Repeated table selectors and backtick quoting + +>>> run_pipeline --refresh table:table1 --refresh table2 +• Refreshed [table1, table2] >>> print_requests.py //pipelines { "method": "POST", - "path": "/api/2.0/pipelines/[UUID][0]/updates", + "path": "/api/2.0/pipelines/[UUID]/updates", "body": { - "full_refresh_selection": [ + "development": true, + "refresh_selection": [ "table1", "table2" - ], - "refresh_selection": [ - "table3", - "table4" ] } } + +>>> run_pipeline --full-refresh table:`table1` `table2` +• Full refreshed [`table1`, `table2`] + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "full_refresh_selection": [ + "`table1`", + "`table2`" + ] + } +} + +=== Selecting multiple source files + +>>> run_pipeline --refresh file:`pipeline_file.py` file:src/pipeline_notebook.sql +• Refreshed tables defined in [/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py, /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/src/pipeline_notebook] + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py", + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/src/pipeline_notebook" + ] + } +} + +>>> print_requests.py //pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "development": true, + "full_refresh_selection_by_path": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/pipeline_file.py", + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/[UNIQUE_NAME]/files/src/pipeline_notebook" + ] + } +} + +=== Rejecting invalid selector syntax before restart +>>> musterr [CLI] pipelines run --refresh table1,table2 --restart --no-wait +Error: invalid argument "table1,table2" for "--refresh" flag: comma-separated selectors are no longer supported; separate selectors with spaces inside quotes, or repeat the flag; use backticks for literal commas +>>> musterr [CLI] pipelines run --full-refresh table1,table2 --restart --no-wait +Error: invalid argument "table1,table2" for "--full-refresh" flag: comma-separated selectors are no longer supported; separate selectors with spaces inside quotes, or repeat the flag; use backticks for literal commas +>>> musterr [CLI] pipelines run --refresh table:`table1 --restart --no-wait +Error: invalid argument "table:`table1" for "--refresh" flag: unclosed backtick in refresh selector +>>> musterr [CLI] pipelines run --refresh --restart --no-wait +Error: invalid argument "" for "--refresh" flag: expected at least one table or file selector + +>>> musterr [CLI] pipelines run --refresh flow:ingest --restart --no-wait +Error: invalid --refresh: flow selectors are not supported yet; select tables or files + +>>> print_requests.py //pipelines diff --git a/acceptance/pipelines/run/refresh-flags/pipeline_file.py b/acceptance/pipelines/run/refresh-flags/pipeline_file.py index aa0c44e3f6d..51dcf9ed5d7 100644 --- a/acceptance/pipelines/run/refresh-flags/pipeline_file.py +++ b/acceptance/pipelines/run/refresh-flags/pipeline_file.py @@ -1,7 +1,14 @@ import dlt from pyspark.sql import SparkSession +spark = SparkSession.builder.getOrCreate() + @dlt.table -def my_table(): +def table1(): return spark.range(10) + + +@dlt.table +def table2(): + return spark.range(20) diff --git a/acceptance/pipelines/run/refresh-flags/script b/acceptance/pipelines/run/refresh-flags/script index b324074d59f..e54639bf000 100644 --- a/acceptance/pipelines/run/refresh-flags/script +++ b/acceptance/pipelines/run/refresh-flags/script @@ -1,18 +1,106 @@ +export BUNDLE_VAR_unique_name="$UNIQUE_NAME" +cleanup() { + $CLI bundle destroy --auto-approve &> LOG.cleanup +} +trap cleanup EXIT + +run_log="$PWD/LOG.run" + +# Cloud event logs and durations vary between runs. +run_pipeline() { + MSYS_NO_PATHCONV=1 $CLI pipelines run "$@" &> "$run_log" + contains.py "completed successfully." < "$run_log" > "$run_log.checked" + grep -E '^• (All tables are |Refreshed |Full refreshed )' "$run_log" +} + trace $CLI pipelines deploy +remote=$($CLI bundle summary -o json | jq -r .workspace.file_path) rm "$OUT_REQUESTS" -title "Running pipeline with --refresh flag and specific tables" -trace $CLI pipelines run --refresh table1,table2 +title "Initialize all tables with --full-refresh-all" +trace run_pipeline --full-refresh-all trace print_requests.py //pipelines -title "Running pipeline with --full-refresh-all flag" -trace $CLI pipelines run --full-refresh-all +title "Running pipeline with --refresh flag and specific tables" +trace run_pipeline --refresh "table1 table2" trace print_requests.py //pipelines title "Running pipeline with --full-refresh flag and specific tables" -trace $CLI pipelines run --full-refresh table1,table2 +trace run_pipeline --full-refresh "table1 table2" trace print_requests.py //pipelines title "Running pipeline with --full-refresh flag and --refresh flag" -trace $CLI pipelines run --full-refresh table1,table2 --refresh table3,table4 +trace run_pipeline --full-refresh "table:table1 table:table2" --refresh "table:table3 table:table4" +trace print_requests.py //pipelines + +title "Refreshing tables and fully refreshing a source file together" +trace run_pipeline --refresh "table:table3 table:table4" --full-refresh file:pipeline_file.py +trace print_requests.py //pipelines + +title "Refreshing tables defined in a source file" +trace run_pipeline --refresh file:pipeline_file.py +trace print_requests.py //pipelines + +title "Fully refreshing tables defined in a source file" +trace run_pipeline --full-refresh file:pipeline_file.py +trace print_requests.py //pipelines + +title "Refreshing a source notebook" +trace run_pipeline --refresh file:src/pipeline_notebook.sql +trace print_requests.py //pipelines + +title "Refreshing an absolute workspace path" +trace run_pipeline --refresh "file:$remote/pipeline_file.py" +trace print_requests.py //pipelines + +title "Refreshing from a subdirectory" +trace withdir src run_pipeline --refresh file:../pipeline_file.py +trace print_requests.py //pipelines + +title "Bundle run supports both file flags" +trace $CLI bundle run my_pipeline --refresh file:pipeline_file.py --full-refresh file:src/pipeline_notebook.sql &> LOG.bundle +trace print_requests.py //pipelines + +title "Refreshing a source file and fully refreshing tables together" +trace run_pipeline --refresh file:pipeline_file.py --full-refresh "table:table3 table:table4" +trace print_requests.py //pipelines + +title "Rejecting missing sources and incompatible refresh modes" +trace musterr $CLI pipelines run --refresh table: --restart --no-wait +trace musterr $CLI pipelines run --full-refresh file: --restart --no-wait +trace musterr $CLI pipelines run --refresh file:./missing.sql --no-wait +trace musterr $CLI pipelines run --refresh file:pipeline_file.py --full-refresh-all --no-wait +trace musterr $CLI pipelines run --full-refresh file:./missing.sql --restart --no-wait +touch "$OUT_REQUESTS" +trace print_requests.py //pipelines + +title "Rejecting mixed selectors before restarting the pipeline" +trace musterr $CLI pipelines run --refresh "table:table1 file:pipeline_file.py" --restart --no-wait +trace musterr $CLI pipelines run --full-refresh "table:table2 file:pipeline_file.py" --restart --no-wait +touch "$OUT_REQUESTS" +trace print_requests.py //pipelines + +trace musterr $CLI bundle run my_pipeline --refresh "table:table1 file:pipeline_file.py" --restart --no-wait +touch "$OUT_REQUESTS" +trace print_requests.py //pipelines + +title "Repeated table selectors and backtick quoting" +trace run_pipeline --refresh table:table1 --refresh table2 +trace print_requests.py //pipelines +trace run_pipeline --full-refresh 'table:`table1` `table2`' +trace print_requests.py //pipelines + +title "Selecting multiple source files" +trace run_pipeline --refresh 'file:`pipeline_file.py` file:src/pipeline_notebook.sql' +trace print_requests.py //pipelines +trace $CLI bundle run my_pipeline --full-refresh 'file:pipeline_file.py file:src/pipeline_notebook.sql' &> LOG.bundle.files +trace print_requests.py //pipelines + +title "Rejecting invalid selector syntax before restart" +trace musterr $CLI pipelines run --refresh table1,table2 --restart --no-wait 2>&1 | grep -E '^(>>> |Error:)' +trace musterr $CLI pipelines run --full-refresh table1,table2 --restart --no-wait 2>&1 | grep -E '^(>>> |Error:)' +trace musterr $CLI pipelines run --refresh 'table:`table1' --restart --no-wait 2>&1 | grep -E '^(>>> |Error:)' +trace musterr $CLI pipelines run --refresh '' --restart --no-wait 2>&1 | grep -E '^(>>> |Error:)' +trace musterr $CLI pipelines run --refresh flow:ingest --restart --no-wait +touch "$OUT_REQUESTS" trace print_requests.py //pipelines diff --git a/acceptance/pipelines/run/refresh-flags/src/pipeline_notebook.sql b/acceptance/pipelines/run/refresh-flags/src/pipeline_notebook.sql new file mode 100644 index 00000000000..be956582f1c --- /dev/null +++ b/acceptance/pipelines/run/refresh-flags/src/pipeline_notebook.sql @@ -0,0 +1,6 @@ +-- Databricks notebook source +CREATE OR REFRESH MATERIALIZED VIEW table3 AS SELECT 3 AS id; + +-- COMMAND ---------- + +CREATE OR REFRESH MATERIALIZED VIEW table4 AS SELECT 4 AS id; diff --git a/acceptance/pipelines/run/refresh-flags/test.toml b/acceptance/pipelines/run/refresh-flags/test.toml index 159efe02696..ff9f96e2553 100644 --- a/acceptance/pipelines/run/refresh-flags/test.toml +++ b/acceptance/pipelines/run/refresh-flags/test.toml @@ -1 +1,2 @@ +Cloud = true RecordRequests = true diff --git a/bundle/bundle.go b/bundle/bundle.go index cb615987468..9f686a7e466 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -397,6 +397,28 @@ func (b *Bundle) InternalDir(ctx context.Context) (string, error) { return dir, nil } +// GetSyncFiles lists files matching the bundle sync configuration. +func (b *Bundle) GetSyncFiles(ctx context.Context) ([]fileset.File, error) { + includes, err := b.GetSyncIncludePatterns(ctx) + if err != nil { + return nil, err + } + list, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, includes, b.Config.Sync.Exclude) + if err != nil { + return nil, err + } + return list.Files(ctx) +} + +// RemoteFilePath returns the workspace root for bundle files: +// the sync root for in-place deployments, otherwise workspace.file_path. +func (b *Bundle) RemoteFilePath() string { + if b.IsSourceLinkedDeployment() { + return filepath.ToSlash(b.SyncRootPath) + } + return b.Config.Workspace.FilePath +} + // GetSyncIncludePatterns returns a list of user defined includes // And also adds InternalDir folder to include list for sync command // so this folder is always synced diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 35cffd0dcbe..32dd9c90ab7 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -15,7 +15,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - libsync "github.com/databricks/cli/libs/sync" ) // Every diagnostic below is reported against an on_file_change entry. @@ -85,15 +84,7 @@ func listSyncableRelPaths(ctx context.Context, b *bundle.Bundle) ([]string, erro // additionally drops files whose notebook type it cannot determine, such as a // malformed .ipynb. Those stay watched on purpose - editing one is exactly when the // run should re-fire - so a pattern can hash a file that this deploy will not upload. - includes, err := b.GetSyncIncludePatterns(ctx) - if err != nil { - return nil, err - } - fl, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, includes, b.Config.Sync.Exclude) - if err != nil { - return nil, err - } - files, err := fl.Files(ctx) + files, err := b.GetSyncFiles(ctx) if err != nil { return nil, err } diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index a5e0d3bf086..b8245569981 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -118,7 +118,7 @@ func ResolveVolumePathReferencesOnlyResources() bundle.Mutator { func lookup(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) { if b.IsSourceLinkedDeployment() { if path.String() == "workspace.file_path" { - return dyn.V(b.SyncRootPath), nil + return dyn.V(b.RemoteFilePath()), nil } } // Future opportunity: if we lookup this path in both the given root diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 956d010ec95..a2a32d14c06 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -15,7 +15,6 @@ import ( "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/notebook" @@ -320,8 +319,7 @@ func (t *translateContext) rewriteValue(ctx context.Context, p dyn.Path, v dyn.V } func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContext, translations []func(context.Context, dyn.Value) (dyn.Value, error)) diag.Diagnostics { - switch { - case b.IsImmutableFolder(): + if b.IsImmutableFolder() { // Reject an explicit workspace.file_path: immutable bundles set it // automatically to the content-addressed snapshot location. A user-supplied // value would be silently discarded during path translation, so we error early. @@ -332,12 +330,8 @@ func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContex Locations: []diag.Location{loc}, }} } - t.remoteRoot = resources.SnapshotFullPathRef + "/files" - case t.b.IsSourceLinkedDeployment(): - t.remoteRoot = t.b.SyncRootPath - default: - t.remoteRoot = t.b.Config.Workspace.FilePath } + t.remoteRoot = b.RemoteFilePath() err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { var err error diff --git a/bundle/deploy/metadata/compute.go b/bundle/deploy/metadata/compute.go index 9cc260f7b91..86406fbd5d8 100644 --- a/bundle/deploy/metadata/compute.go +++ b/bundle/deploy/metadata/compute.go @@ -128,10 +128,9 @@ func (m *compute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics b.Metadata.Config.Resources.Dashboards = dashboardsMetadata // Set file upload destination of the bundle in metadata - b.Metadata.Config.Workspace.FilePath = b.Config.Workspace.FilePath + b.Metadata.Config.Workspace.FilePath = b.RemoteFilePath() // In source-linked deployment files are not copied and resources use source files, therefore we use sync path as file path in metadata if b.IsSourceLinkedDeployment() { - b.Metadata.Config.Workspace.FilePath = b.SyncRootPath b.Metadata.Config.Presets.SourceLinkedDeployment = true } diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go index 97b8144d28b..ad0860c3356 100644 --- a/bundle/run/pipeline.go +++ b/bundle/run/pipeline.go @@ -102,6 +102,25 @@ func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (output.RunOutp if err != nil { return nil, err } + var hasQuotedTables bool + for _, tables := range [][]string{req.RefreshSelection, req.FullRefreshSelection} { + for _, table := range tables { + hasQuotedTables = hasQuotedTables || strings.ContainsRune(table, '`') + } + } + if hasQuotedTables { + pipeline, err := w.Pipelines.GetByPipelineId(ctx, pipelineID) + if err != nil { + return nil, err + } + // Legacy mode takes raw names; default publishing mode parses SQL identifiers. + // Unset modes follow the backend's schema-based legacy detection. + if pipeline.EffectivePublishingMode == pipelines.PublishingModeLegacyPublishingMode || + (pipeline.EffectivePublishingMode == "" && pipeline.Spec.Schema == "") { + req.RefreshSelection = unquoteRefreshTables(req.RefreshSelection) + req.FullRefreshSelection = unquoteRefreshTables(req.FullRefreshSelection) + } + } res, err := w.Pipelines.StartUpdate(ctx, *req) if err != nil { @@ -185,6 +204,18 @@ func (r *pipelineRunner) Cancel(ctx context.Context) error { } func (r *pipelineRunner) Restart(ctx context.Context, opts *Options) (output.RunOutput, error) { + hasFiles := len(opts.Pipeline.RefreshByPath) > 0 || len(opts.Pipeline.FullRefreshByPath) > 0 + // Check before stopping the active update. + if hasFiles { + pipeline, err := r.bundle.WorkspaceClient(ctx).Pipelines.GetByPipelineId(ctx, r.pipeline.ID) + if err != nil { + return nil, err + } + if pipeline.Spec.Continuous { //nolint:staticcheck // Direct pipeline runs honor this setting. + return nil, errors.New("file selectors with --restart require a triggered pipeline") + } + } + sp := cmdio.NewSpinner(ctx) sp.Update("Cancelling the active pipeline update") err := r.Cancel(ctx) diff --git a/bundle/run/pipeline_options.go b/bundle/run/pipeline_options.go index 63ff3e81289..51f187741b4 100644 --- a/bundle/run/pipeline_options.go +++ b/bundle/run/pipeline_options.go @@ -1,7 +1,9 @@ package run import ( + "errors" "fmt" + "slices" "strings" "github.com/databricks/cli/bundle/config/resources" @@ -14,27 +16,55 @@ type PipelineOptions struct { // Perform a full graph update. RefreshAll bool - // List of tables to update. + // Refresh holds CLI selectors before resolution and table names afterward. Refresh []string + // Workspace file paths resolved from Refresh. + RefreshByPath []string + // Perform a full graph reset and recompute. FullRefreshAll bool - // List of tables to reset and recompute. + // FullRefresh holds CLI selectors before resolution and table names afterward. FullRefresh []string + // Workspace file paths resolved from FullRefresh. + FullRefreshByPath []string + // Perform an update to validate graph correctness. ValidateOnly bool } func (o *PipelineOptions) Define(fs *flag.FlagSet) { fs.BoolVar(&o.RefreshAll, "refresh-all", false, "Perform a full graph update.") - fs.StringSliceVar(&o.Refresh, "refresh", nil, "List of tables to update.") - fs.BoolVar(&o.FullRefreshAll, "full-refresh-all", false, "Perform a full graph reset and recompute.") - fs.StringSliceVar(&o.FullRefresh, "full-refresh", nil, "List of tables to reset and recompute.") + o.DefineRefreshFlags(fs) fs.BoolVar(&o.ValidateOnly, "validate-only", false, "Perform an update to validate graph correctness.") } +// PipelineRefreshHelp describes table and file refresh selectors. +const PipelineRefreshHelp = `For pipeline runs, --refresh and --full-refresh accept: + Tables: orders or table:orders + Source files: file:./src/pipeline.py (file: is required) + +Put multiple selectors in quotes, or repeat the flag: + --refresh "orders payments" + --refresh "file:./src/a.py file:./src/b.py" + --refresh table:orders --refresh table:payments + +For a path containing spaces: + --refresh 'file:` + "`./src/my pipeline.py`" + `' + +Each refresh mode accepts either tables or files. +Local file paths are relative to the current directory and map to deployed files. +Absolute /Workspace/ paths are also accepted. +Deploy local changes before running.` + +func (o *PipelineOptions) DefineRefreshFlags(fs *flag.FlagSet) { + fs.Var((*refreshSelectionFlag)(&o.Refresh), "refresh", "Space-separated table: or file: selectors to update. Quote lists or repeat the flag.") + fs.BoolVar(&o.FullRefreshAll, "full-refresh-all", false, "Perform a full graph reset and recompute.") + fs.Var((*refreshSelectionFlag)(&o.FullRefresh), "full-refresh", "Space-separated table: or file: selectors to reset and recompute. Quote lists or repeat the flag.") +} + // Validate returns if the combination of options is valid. func (o *PipelineOptions) Validate(pipeline *resources.Pipeline) error { var mutuallyExclusiveFlags []string @@ -51,10 +81,18 @@ func (o *PipelineOptions) Validate(pipeline *resources.Pipeline) error { if len(mutuallyExclusiveFlags) > 1 { return fmt.Errorf("%s pipeline run flags are mutually exclusive", strings.Join(mutuallyExclusiveFlags, ", ")) - } else if len(mutuallyExclusiveFlags) == 1 && (len(o.Refresh) > 0 || len(o.FullRefresh) > 0) { + } else if len(mutuallyExclusiveFlags) == 1 && (len(o.Refresh) > 0 || len(o.FullRefresh) > 0 || len(o.RefreshByPath) > 0 || len(o.FullRefreshByPath) > 0) { return fmt.Errorf("cannot use --refresh or --full-refresh together with %s, these flags are mutually exclusive", strings.Join(mutuallyExclusiveFlags, "")) } + // The API ignores file selections when table names are supplied for the same refresh mode. + if len(o.Refresh) > 0 && len(o.RefreshByPath) > 0 { + return errors.New("cannot mix table names and source files in --refresh; use only table names or only source files") + } + if len(o.FullRefresh) > 0 && len(o.FullRefreshByPath) > 0 { + return errors.New("cannot mix table names and source files in --full-refresh; use only table names or only source files") + } + return nil } @@ -66,10 +104,12 @@ func (o *PipelineOptions) toPayload(pipeline *resources.Pipeline, pipelineID str PipelineId: pipelineID, // Note: `RefreshAll` is implied if the fields below are not set. - RefreshSelection: o.Refresh, - FullRefresh: o.FullRefreshAll, - FullRefreshSelection: o.FullRefresh, - ValidateOnly: o.ValidateOnly, + RefreshSelection: o.Refresh, + FullRefresh: o.FullRefreshAll, + FullRefreshSelection: o.FullRefresh, + RefreshSelectionByPath: o.RefreshByPath, + FullRefreshSelectionByPath: o.FullRefreshByPath, + ValidateOnly: o.ValidateOnly, // Only true is sent. When false the field is omitted, so the backend keeps // using the pipeline-level development property while it is being deprecated. @@ -77,3 +117,11 @@ func (o *PipelineOptions) toPayload(pipeline *resources.Pipeline, pipelineID str } return payload, nil } + +func unquoteRefreshTables(tables []string) []string { + tables = slices.Clone(tables) + for i, table := range tables { + tables[i] = unquoteRefreshSelector(table) + } + return tables +} diff --git a/bundle/run/pipeline_options_test.go b/bundle/run/pipeline_options_test.go index 1eda4c8bae7..164f731f0ba 100644 --- a/bundle/run/pipeline_options_test.go +++ b/bundle/run/pipeline_options_test.go @@ -1,13 +1,86 @@ package run import ( + "encoding/json" "testing" + "github.com/databricks/cli/bundle/config/resources" flag "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func TestPipelineOptionsPathSelectionsPayload(t *testing.T) { + tests := []struct { + name string + opts PipelineOptions + expected string + }{ + { + name: "file selections for both refresh modes", + opts: PipelineOptions{ + RefreshByPath: []string{"/Workspace/Shared/source.sql"}, + FullRefreshByPath: []string{"/Workspace/Shared/notebook"}, + }, + expected: `{"refresh_selection_by_path":["/Workspace/Shared/source.sql"],"full_refresh_selection_by_path":["/Workspace/Shared/notebook"]}`, + }, + { + name: "table refresh and file full-refresh", + opts: PipelineOptions{ + Refresh: []string{"catalog.schema.table1"}, + FullRefreshByPath: []string{"/Workspace/Shared/notebook"}, + }, + expected: `{"refresh_selection":["catalog.schema.table1"],"full_refresh_selection_by_path":["/Workspace/Shared/notebook"]}`, + }, + { + name: "file refresh and table full-refresh", + opts: PipelineOptions{ + RefreshByPath: []string{"/Workspace/Shared/source.sql"}, + FullRefresh: []string{"catalog.schema.table2"}, + }, + expected: `{"refresh_selection_by_path":["/Workspace/Shared/source.sql"],"full_refresh_selection":["catalog.schema.table2"]}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + payload, err := tt.opts.toPayload(&resources.Pipeline{}, "pipeline-id", false) + require.NoError(t, err) + data, err := json.Marshal(payload) + require.NoError(t, err) + assert.JSONEq(t, tt.expected, string(data)) + }) + } +} + +func TestPipelineOptionsRejectMixedSelectors(t *testing.T) { + tests := []struct { + name string + opts PipelineOptions + }{ + { + name: "--refresh", + opts: PipelineOptions{ + Refresh: []string{"table1"}, + RefreshByPath: []string{"/Workspace/Shared/source.sql"}, + }, + }, + { + name: "--full-refresh", + opts: PipelineOptions{ + FullRefresh: []string{"table1"}, + FullRefreshByPath: []string{"/Workspace/Shared/source.sql"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + payload, err := tt.opts.toPayload(nil, "pipeline-id", false) + require.ErrorContains(t, err, "cannot mix table names and source files in "+tt.name) + assert.Nil(t, payload) + }) + } +} + func setupPipelineOptions(t *testing.T) (*flag.FlagSet, *PipelineOptions) { var fs flag.FlagSet var opts PipelineOptions @@ -24,7 +97,7 @@ func TestPipelineOptionsRefreshAll(t *testing.T) { func TestPipelineOptionsRefresh(t *testing.T) { fs, opts := setupPipelineOptions(t) - err := fs.Parse([]string{`--refresh=arg1,arg2,arg3`}) + err := fs.Parse([]string{`--refresh=arg1 arg2 arg3`}) require.NoError(t, err) assert.Equal(t, []string{"arg1", "arg2", "arg3"}, opts.Refresh) } @@ -38,7 +111,7 @@ func TestPipelineOptionsFullRefreshAll(t *testing.T) { func TestPipelineOptionsFullRefresh(t *testing.T) { fs, opts := setupPipelineOptions(t) - err := fs.Parse([]string{`--full-refresh=arg1,arg2,arg3`}) + err := fs.Parse([]string{`--full-refresh=arg1 arg2 arg3`}) require.NoError(t, err) assert.Equal(t, []string{"arg1", "arg2", "arg3"}, opts.FullRefresh) } @@ -53,9 +126,9 @@ func TestPipelineOptionsValidateOnly(t *testing.T) { func TestPipelineOptionsValidateSuccessWithSingleOption(t *testing.T) { args := []string{ `--refresh-all`, - `--refresh=arg1,arg2,arg3`, + `--refresh=arg1 arg2 arg3`, `--full-refresh-all`, - `--full-refresh=arg1,arg2,arg3`, + `--full-refresh=arg1 arg2 arg3`, `--validate-only`, } for _, arg := range args { @@ -69,7 +142,7 @@ func TestPipelineOptionsValidateSuccessWithSingleOption(t *testing.T) { func TestPipelineOptionsValidateSuccessWithRefreshAndFullRefresh(t *testing.T) { fs, opts := setupPipelineOptions(t) - err := fs.Parse([]string{`--refresh=arg1,arg2`, `--full-refresh=arg3,arg4`}) + err := fs.Parse([]string{`--refresh=arg1 arg2`, `--full-refresh=arg3 arg4`}) require.NoError(t, err) err = opts.Validate(nil) assert.NoError(t, err) @@ -115,6 +188,11 @@ func TestPipelineOptionsValidateFailureWithRefreshAndMutuallyExclusiveFlags(t *t require.NoError(t, err) err = opts.Validate(nil) assert.ErrorContains(t, err, "cannot use --refresh or --full-refresh together with") + + opts.RefreshByPath, opts.Refresh = opts.Refresh, nil + opts.FullRefreshByPath, opts.FullRefresh = opts.FullRefresh, nil + err = opts.Validate(nil) + assert.ErrorContains(t, err, "cannot use --refresh or --full-refresh together with") } } } diff --git a/bundle/run/pipeline_refresh.go b/bundle/run/pipeline_refresh.go new file mode 100644 index 00000000000..af2e75a37af --- /dev/null +++ b/bundle/run/pipeline_refresh.go @@ -0,0 +1,232 @@ +package run + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + "unicode" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/notebook" + "github.com/databricks/cli/libs/sync" +) + +type refreshSelectionFlag []string + +func (f *refreshSelectionFlag) String() string { + return strings.Join(*f, " ") +} + +func (f *refreshSelectionFlag) Type() string { + return "selectors" +} + +func (f *refreshSelectionFlag) Set(value string) error { + var selectors []string + start := -1 + quoted := false + for i, c := range value { + if !quoted && unicode.IsSpace(c) { + if start >= 0 { + selectors = append(selectors, value[start:i]) + start = -1 + } + continue + } + if start < 0 { + start = i + } + switch c { + case '`': + quoted = !quoted + case ',': + if !quoted { + return errors.New("comma-separated selectors are no longer supported; separate selectors with spaces inside quotes, or repeat the flag; use backticks for literal commas") + } + } + } + if quoted { + return errors.New("unclosed backtick in refresh selector") + } + if start >= 0 { + selectors = append(selectors, value[start:]) + } + if len(selectors) == 0 { + return errors.New("expected at least one table or file selector") + } + *f = append(*f, selectors...) + return nil +} + +func unquoteRefreshSelector(value string) string { + var result strings.Builder + quoted := false + for i := 0; i < len(value); i++ { + if value[i] != '`' { + result.WriteByte(value[i]) + continue + } + if quoted && i+1 < len(value) && value[i+1] == '`' { + result.WriteByte('`') + i++ + } else { + quoted = !quoted + } + } + return result.String() +} + +// ResolveRefreshSelectors separates table names and workspace file paths, then validates flag combinations. +func (o *PipelineOptions) ResolveRefreshSelectors(ctx context.Context, b *bundle.Bundle) error { + r := refreshSelectorResolver{bundle: b} + var err error + o.Refresh, o.RefreshByPath, err = r.resolve(ctx, o.Refresh) + if err != nil { + return fmt.Errorf("invalid --refresh: %w", err) + } + o.FullRefresh, o.FullRefreshByPath, err = r.resolve(ctx, o.FullRefresh) + if err != nil { + return fmt.Errorf("invalid --full-refresh: %w", err) + } + // Validate before Restart stops an active update. + return o.Validate(nil) +} + +type refreshSelectorResolver struct { + bundle *bundle.Bundle + remoteNames map[string]string +} + +func (r *refreshSelectorResolver) resolve(ctx context.Context, selectors []string) (tables, paths []string, err error) { + b := r.bundle + for _, selector := range selectors { + if selector == "``" { + return nil, nil, errors.New("table selector requires a table name") + } + if table, ok := strings.CutPrefix(selector, "table:"); ok { + if unquoteRefreshSelector(table) == "" { + return nil, nil, errors.New("table: requires a table name") + } + tables = append(tables, table) + continue + } + file, ok := strings.CutPrefix(selector, "file:") + if !ok { + if strings.HasPrefix(selector, "flow:") { + return nil, nil, errors.New("flow selectors are not supported yet; select tables or files") + } + tables = append(tables, selector) + continue + } + file = unquoteRefreshSelector(file) + if file == "" { + return nil, nil, errors.New("file: requires a source file path") + } + selector = file + // /Workspace/ distinguishes remote paths from local paths like /Users/... on macOS. + if strings.HasPrefix(selector, "/Workspace/") { + if b.IsSourceLinkedDeployment() { + rel, err := filepath.Rel(b.SyncRootPath, filepath.FromSlash(selector)) + if err == nil && filepath.IsLocal(rel) { + remotePath, err := r.resolveInPlacePath(selector, rel) + if err == nil { + paths = append(paths, remotePath) + continue + } + if !errors.Is(err, fs.ErrNotExist) { + return nil, nil, err + } + } + } + paths = append(paths, selector) + continue + } + + info, statErr := os.Stat(selector) + if errors.Is(statErr, fs.ErrNotExist) { + return nil, nil, fmt.Errorf("source file %q not found; use an existing source file", selector) + } + if statErr != nil { + return nil, nil, fmt.Errorf("cannot read refresh selector %q: %w", selector, statErr) + } + if info.IsDir() { + return nil, nil, fmt.Errorf("refresh selector %q is a directory; select a source file", selector) + } + + localPath, err := filepath.Abs(selector) + if err != nil { + return nil, nil, err + } + rel, err := filepath.Rel(b.SyncRootPath, localPath) + if err != nil { + return nil, nil, err + } + if !filepath.IsLocal(rel) { + return nil, nil, fmt.Errorf("source file %q is outside the bundle sync root", selector) + } + if b.IsSourceLinkedDeployment() { + remotePath, err := r.resolveInPlacePath(selector, rel) + if err != nil { + return nil, nil, err + } + paths = append(paths, remotePath) + continue + } + if r.remoteNames == nil { + // Refresh uses the deployed snapshot. + if b.IsImmutableFolder() { + if err := b.DeploymentBundle.ResolveConfigAgainstState(&b.Config, dyn.MustPathFromString("workspace.file_path")); err != nil { + return nil, nil, err + } + } + r.remoteNames, err = refreshSourcePaths(ctx, b) + if err != nil { + return nil, nil, err + } + } + remoteName, ok := r.remoteNames[filepath.ToSlash(rel)] + if !ok { + if _, _, err := notebook.DetectWithFS(b.SyncRoot, filepath.ToSlash(rel)); err != nil { + return nil, nil, fmt.Errorf("cannot identify source file %q: %w", selector, err) + } + return nil, nil, fmt.Errorf("source file %q is excluded from bundle sync; include it and deploy first", selector) + } + remoteRoot := b.RemoteFilePath() + if strings.Contains(remoteRoot, "${") { + return nil, nil, fmt.Errorf("cannot resolve the deployed path for %q; use its absolute /Workspace/ path", selector) + } + paths = append(paths, path.Join(remoteRoot, remoteName)) + } + return tables, paths, nil +} + +func (r *refreshSelectorResolver) resolveInPlacePath(selector, rel string) (string, error) { + remoteName := filepath.ToSlash(rel) + isNotebook, _, err := notebook.DetectWithFS(r.bundle.SyncRoot, remoteName) + if err != nil { + return "", fmt.Errorf("cannot identify source file %q: %w", selector, err) + } + if isNotebook { + remoteName = notebook.StripExtension(remoteName) + } + return path.Join(r.bundle.RemoteFilePath(), remoteName), nil +} + +// refreshSourcePaths maps local files to their deployed names. +func refreshSourcePaths(ctx context.Context, b *bundle.Bundle) (map[string]string, error) { + files, err := b.GetSyncFiles(ctx) + if err != nil { + return nil, err + } + state, err := sync.NewSnapshotState(files) + if err != nil { + return nil, err + } + return state.LocalToRemoteNames, nil +} diff --git a/bundle/run/pipeline_refresh_test.go b/bundle/run/pipeline_refresh_test.go new file mode 100644 index 00000000000..6dcefea335b --- /dev/null +++ b/bundle/run/pipeline_refresh_test.go @@ -0,0 +1,312 @@ +package run + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/libs/vfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRefreshSelectionFlag(t *testing.T) { + var manyTables, manyFiles []string + for i := range 50 { + manyTables = append(manyTables, fmt.Sprintf("table_%d", i)) + manyFiles = append(manyFiles, fmt.Sprintf("file:src/pipeline_%d.py", i)) + } + tests := []struct { + name string + values []string + want []string + err string + }{ + {name: "50 tables", values: []string{strings.Join(manyTables, " ")}, want: manyTables}, + {name: "50 files", values: []string{strings.Join(manyFiles, " ")}, want: manyFiles}, + {name: "single table", values: []string{"orders"}, want: []string{"orders"}}, + {name: "table list", values: []string{"orders payments"}, want: []string{"orders", "payments"}}, + {name: "repeated flag", values: []string{"orders payments", "table:customers"}, want: []string{"orders", "payments", "table:customers"}}, + {name: "file list", values: []string{"file:a.py file:b.py"}, want: []string{"file:a.py", "file:b.py"}}, + {name: "whitespace", values: []string{" orders\tpayments\ncustomers\u2003returns "}, want: []string{"orders", "payments", "customers", "returns"}}, + {name: "quoted table", values: []string{"`order details` payments"}, want: []string{"`order details`", "payments"}}, + {name: "qualified identifier", values: []string{"table:`catalog`.`schema`.`order details`"}, want: []string{"table:`catalog`.`schema`.`order details`"}}, + {name: "quoted comma", values: []string{"table:`a,b` orders"}, want: []string{"table:`a,b`", "orders"}}, + {name: "escaped backtick", values: []string{"table:`a`` b,c` orders"}, want: []string{"table:`a`` b,c`", "orders"}}, + {name: "quoted file", values: []string{"file:`./my pipeline.py` file:b.py"}, want: []string{"file:`./my pipeline.py`", "file:b.py"}}, + {name: "empty value", values: []string{""}, err: "expected at least one"}, + {name: "whitespace only", values: []string{" \t "}, err: "expected at least one"}, + {name: "old comma list", values: []string{"orders,payments"}, err: "comma-separated selectors are no longer supported"}, + {name: "comma after quoted name", values: []string{"`orders`,payments"}, err: "comma-separated selectors"}, + {name: "unclosed backtick", values: []string{"orders `payments"}, err: "unclosed backtick"}, + {name: "unclosed escaped backtick", values: []string{"file:`a``"}, err: "unclosed backtick"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var flag refreshSelectionFlag + for _, value := range tt.values { + err := flag.Set(value) + if tt.err != "" { + require.ErrorContains(t, err, tt.err) + assert.Empty(t, flag) + return + } + require.NoError(t, err) + } + assert.Equal(t, tt.want, []string(flag)) + }) + } +} + +func TestResolveRefreshSelectors(t *testing.T) { + longTable := strings.Repeat("c", 100) + "." + strings.Repeat("s", 100) + "." + strings.Repeat("t", 100) + tests := []struct { + name string + selectors []string + tables []string + paths []string + err string + }{ + {name: "no selection"}, + {name: "long qualified table", selectors: []string{longTable}, tables: []string{longTable}}, + { + name: "table identifiers", + selectors: []string{"table1", "schema.table2", "schema.sql", "catalog.schema.sql", "`schema`.`table/with/slashes`", "`schema`.`sql`"}, + tables: []string{"table1", "schema.table2", "schema.sql", "catalog.schema.sql", "`schema`.`table/with/slashes`", "`schema`.`sql`"}, + }, + { + name: "bare table with the same name as a file", + selectors: []string{"orders"}, + tables: []string{"orders"}, + }, + { + name: "mixed table and file selectors", + selectors: []string{"table:table1", "file:src/query.sql", "file:src/module.py"}, + tables: []string{"table1"}, + paths: []string{"/Workspace/Shared/project/files/src/query.sql", "/Workspace/Shared/project/files/src/module.py"}, + }, + { + name: "notebook extension removed", + selectors: []string{"file:src/notebook.py", "file:src/jupyter.ipynb"}, + paths: []string{"/Workspace/Shared/project/files/src/notebook", "/Workspace/Shared/project/files/src/jupyter"}, + }, + { + name: "workspace path", + selectors: []string{"file:/Workspace/Shared/remote+part.sql"}, + paths: []string{"/Workspace/Shared/remote+part.sql"}, + }, + { + name: "explicit table identifiers", + selectors: []string{"table:orders", "table:catalog.schema.orders", "table:`file:orders`", "table:file:orders"}, + tables: []string{"orders", "catalog.schema.orders", "`file:orders`", "file:orders"}, + }, + {name: "empty quoted table", selectors: []string{"table:``"}, err: "table: requires a table name"}, + {name: "empty bare table", selectors: []string{"``"}, err: "table selector requires a table name"}, + {name: "empty quoted file", selectors: []string{"file:``"}, err: "file: requires a source file path"}, + {name: "empty table", selectors: []string{"table:"}, err: "table: requires a table name"}, + {name: "unsupported flow", selectors: []string{"flow:ingest"}, err: "flow selectors are not supported"}, + {name: "quoted workspace path", selectors: []string{"file:`/Workspace/Shared/my pipeline.py`"}, paths: []string{"/Workspace/Shared/my pipeline.py"}}, + {name: "escaped backtick in workspace path", selectors: []string{"file:`/Workspace/Shared/a``b.py`"}, paths: []string{"/Workspace/Shared/a`b.py"}}, + {name: "local file with spaces", selectors: []string{"file:`src/my query.sql`"}, paths: []string{"/Workspace/Shared/project/files/src/my query.sql"}}, + {name: "empty path", selectors: []string{"file:"}, err: "file: requires a source file path"}, + {name: "missing relative file", selectors: []string{"file:src/missing.sql"}, err: `source file "src/missing.sql" not found`}, + {name: "explicit missing filename", selectors: []string{"file:./missing.sql"}, err: `source file "./missing.sql" not found`}, + {name: "table named like a directory", selectors: []string{"src"}, tables: []string{"src"}}, + {name: "explicit directory", selectors: []string{"file:./src"}, err: "is a directory"}, + {name: "excluded file", selectors: []string{"file:src/excluded.sql"}, err: "is excluded from bundle sync"}, + {name: "invalid notebook", selectors: []string{"file:src/invalid.ipynb"}, err: "invalid Jupyter notebook file"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "orders"), []byte("SELECT 1;\n"), 0o600)) + require.NoError(t, os.MkdirAll(filepath.Join(root, "src"), 0o700)) + for name, content := range map[string]string{ + "query.sql": "SELECT 1;\n", + "my query.sql": "SELECT 1;\n", + "module.py": "print('file')\n", + "notebook.py": "# Databricks notebook source\nprint('notebook')\n", + "jupyter.ipynb": `{"cells":[],"metadata":{},"nbformat":4,"nbformat_minor":5}`, + "excluded.sql": "SELECT 2;\n", + "invalid.ipynb": `{}`, + } { + require.NoError(t, os.WriteFile(filepath.Join(root, "src", name), []byte(content), 0o600)) + } + t.Chdir(root) + b := refreshTestBundle(root) + b.Config.Sync.Exclude = []string{"src/excluded.sql"} + tables, paths, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), tt.selectors) + if tt.err != "" { + require.ErrorContains(t, err, tt.err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.tables, tables) + assert.Equal(t, tt.paths, paths) + }) + } +} + +func refreshTestBundle(root string) *bundle.Bundle { + return &bundle.Bundle{ + BundleRootPath: root, + SyncRootPath: root, + SyncRoot: vfs.MustNew(root), + WorktreeRoot: vfs.MustNew(root), + Config: config.Root{ + Bundle: config.Bundle{Name: "test", Target: "default"}, + Workspace: config.Workspace{FilePath: "/Workspace/Shared/project/files"}, + Sync: config.Sync{Paths: []string{"."}}, + }, + } +} + +func TestResolveRefreshSelectorsFromSubdirectory(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "project", "src"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(root, "shared.sql"), []byte("SELECT 1;\n"), 0o600)) + t.Chdir(filepath.Join(root, "project", "src")) + b := refreshTestBundle(root) + b.BundleRootPath = filepath.Join(root, "project") + for _, selector := range []string{"../../shared.sql", filepath.Join(root, "shared.sql")} { + tables, paths, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), []string{"file:" + selector}) + require.NoError(t, err) + assert.Empty(t, tables) + assert.Equal(t, []string{"/Workspace/Shared/project/files/shared.sql"}, paths) + } +} + +func TestResolveRefreshSelectorsOutsideSyncRoot(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(root, "project"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(root, "outside.sql"), []byte("SELECT 1;\n"), 0o600)) + t.Chdir(filepath.Join(root, "project")) + _, _, err := (&refreshSelectorResolver{bundle: refreshTestBundle(filepath.Join(root, "project"))}).resolve(t.Context(), []string{"file:../outside.sql"}) + require.ErrorContains(t, err, "is outside the bundle sync root") +} + +func TestResolveRefreshSelectorsDeploymentMode(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "query.sql"), []byte("SELECT 1;\n"), 0o600)) + t.Chdir(root) + enabled, disabled := true, false + localPath := filepath.ToSlash(filepath.Join(root, "query.sql")) + remotePath := "/Workspace/Shared/project/files/query.sql" + tests := []struct { + name string + mode config.SyncMode + sourceLinked *bool + want string + }{ + {name: "default upload", want: remotePath}, + {name: "in-place", mode: config.SyncModeInPlace, want: localPath}, + {name: "legacy source-linked", sourceLinked: &enabled, want: localPath}, + {name: "overwrite overrides legacy setting", mode: config.SyncModeOverwrite, sourceLinked: &enabled, want: remotePath}, + {name: "in-place overrides legacy setting", mode: config.SyncModeInPlace, sourceLinked: &disabled, want: localPath}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := refreshTestBundle(root) + b.Config.Workspace.SyncMode = tt.mode + b.Config.Presets.SourceLinkedDeployment = tt.sourceLinked + tables, paths, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), []string{"file:query.sql"}) + require.NoError(t, err) + assert.Empty(t, tables) + assert.Equal(t, []string{tt.want}, paths) + }) + } +} + +func TestResolveRefreshSelectorsUnresolvedWorkspacePath(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "query.sql"), []byte("SELECT 1;\n"), 0o600)) + t.Chdir(root) + b := refreshTestBundle(root) + b.Config.Workspace.FilePath = "${resources.internal_immutable_snapshots.immutable.full_path}/files" + b.Config.Workspace.SyncMode = config.SyncModeImmutable + enabled := true + b.Config.Presets.SourceLinkedDeployment = &enabled + b.DeploymentBundle.StateDB.OpenWithData("state", dstate.Database{}) + _, _, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), []string{"file:query.sql"}) + require.ErrorContains(t, err, "use its absolute /Workspace/ path") +} + +func TestResolveRefreshSelectorsDeployedSnapshot(t *testing.T) { + root := t.TempDir() + for _, name := range []string{"query.sql", "full_query.sql"} { + require.NoError(t, os.WriteFile(filepath.Join(root, name), []byte("SELECT 2;\n"), 0o600)) + } + t.Chdir(root) + b := refreshTestBundle(root) + b.Config.Workspace.SyncMode = config.SyncModeImmutable + b.Config.Workspace.FilePath = resources.SnapshotFullPathRef + "/files" + b.DeploymentBundle.StateDB.OpenWithData("state", dstate.Database{ + State: map[string]dstate.ResourceEntry{ + resources.SnapshotKey: { + ID: "deployed-snapshot", + State: []byte(`{"full_path":"/Workspace/Shared/snapshots/deployed/snapshot"}`), + }, + }, + }) + opts := PipelineOptions{Refresh: []string{"file:query.sql"}, FullRefresh: []string{"file:full_query.sql"}} + require.NoError(t, opts.ResolveRefreshSelectors(t.Context(), b)) + assert.Empty(t, opts.Refresh) + assert.Empty(t, opts.FullRefresh) + assert.Equal(t, []string{"/Workspace/Shared/snapshots/deployed/snapshot/files/query.sql"}, opts.RefreshByPath) + assert.Equal(t, []string{"/Workspace/Shared/snapshots/deployed/snapshot/files/full_query.sql"}, opts.FullRefreshByPath) +} + +func TestResolveRefreshSelectorsInPlaceExcludedFiles(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "query.sql"), []byte("SELECT 1;\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "notebook.py"), []byte("# Databricks notebook source\nprint('notebook')\n"), 0o600)) + t.Chdir(root) + b := refreshTestBundle(root) + b.Config.Workspace.SyncMode = config.SyncModeInPlace + b.Config.Sync.Exclude = []string{"query.sql", "notebook.py"} + tables, paths, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), []string{"file:query.sql", "file:notebook.py"}) + require.NoError(t, err) + assert.Empty(t, tables) + assert.Equal(t, []string{filepath.ToSlash(filepath.Join(root, "query.sql")), filepath.ToSlash(filepath.Join(root, "notebook"))}, paths) +} + +func TestResolveRefreshSelectorsInPlaceWorkspacePaths(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "notebook.py"), []byte("# Databricks notebook source\nprint('notebook')\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "module.py"), []byte("print('file')\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(root, "invalid.ipynb"), []byte(`{}`), 0o600)) + b := refreshTestBundle(root) + b.SyncRootPath = filepath.FromSlash("/Workspace/Users/test/project") + b.Config.Workspace.SyncMode = config.SyncModeInPlace + b.Config.Sync.Exclude = []string{"notebook.py"} + + for _, tt := range []struct { + name string + path string + want string + err string + }{ + {name: "source notebook", path: "/Workspace/Users/test/project/notebook.py", want: "/Workspace/Users/test/project/notebook"}, + {name: "regular file", path: "/Workspace/Users/test/project/module.py", want: "/Workspace/Users/test/project/module.py"}, + {name: "deployed notebook", path: "/Workspace/Users/test/project/notebook", want: "/Workspace/Users/test/project/notebook"}, + {name: "outside sync root", path: "/Workspace/Users/test/project-other/notebook.py", want: "/Workspace/Users/test/project-other/notebook.py"}, + {name: "invalid notebook", path: "/Workspace/Users/test/project/invalid.ipynb", err: "invalid Jupyter notebook file"}, + } { + t.Run(tt.name, func(t *testing.T) { + tables, paths, err := (&refreshSelectorResolver{bundle: b}).resolve(t.Context(), []string{"file:" + tt.path}) + if tt.err != "" { + require.ErrorContains(t, err, tt.err) + return + } + require.NoError(t, err) + assert.Empty(t, tables) + assert.Equal(t, []string{tt.want}, paths) + }) + } +} diff --git a/bundle/run/pipeline_test.go b/bundle/run/pipeline_test.go index 354216ee490..0e0c84f5657 100644 --- a/bundle/run/pipeline_test.go +++ b/bundle/run/pipeline_test.go @@ -1,6 +1,7 @@ package run import ( + "errors" "testing" "time" @@ -111,3 +112,113 @@ func TestPipelineRunnerRestart(t *testing.T) { _, err := runner.Restart(ctx, &Options{}) require.NoError(t, err) } + +func TestPipelineRunnerQuotedTables(t *testing.T) { + tests := []struct { + name string + mode pipelines.PublishingMode + schema string + legacy bool + }{ + {name: "DPM with schema", mode: pipelines.PublishingModeDefaultPublishingMode, schema: "default"}, + {name: "DPM with target", mode: pipelines.PublishingModeDefaultPublishingMode}, + {name: "legacy", mode: pipelines.PublishingModeLegacyPublishingMode, legacy: true}, + {name: "unset mode with schema", schema: "default"}, + {name: "unset mode with target", legacy: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fs, opts := setupPipelineOptions(t) + require.NoError(t, fs.Parse([]string{"--refresh", "table:`a`` b,c` orders", "--full-refresh", "`daily rollup`"})) + require.NoError(t, opts.ResolveRefreshSelectors(t.Context(), nil)) + pipeline := &resources.Pipeline{ID: "123"} + pipeline.Schema = tt.schema + b := &bundle.Bundle{} + m := mocks.NewMockWorkspaceClient(t) + m.WorkspaceClient.Config = &sdk_config.Config{Host: "https://test.test"} + b.SetWorkpaceClient(m.WorkspaceClient) + spec := &pipelines.PipelineSpec{Schema: tt.schema} + if tt.schema == "" { + spec.Target = "default" + } + api := m.GetMockPipelinesAPI() + api.EXPECT().GetByPipelineId(mock.Anything, "123").Return(&pipelines.GetPipelineResponse{ + Spec: spec, EffectivePublishingMode: tt.mode, + }, nil).Once() + want := pipelines.StartUpdate{ + PipelineId: "123", RefreshSelection: []string{"`a`` b,c`", "orders"}, FullRefreshSelection: []string{"`daily rollup`"}, + } + if tt.legacy { + want.RefreshSelection = []string{"a` b,c", "orders"} + want.FullRefreshSelection = []string{"daily rollup"} + } + api.EXPECT().StartUpdate(mock.Anything, want).Return(&pipelines.StartUpdateResponse{UpdateId: "456"}, nil).Once() + runner := pipelineRunner{key: "test", bundle: b, pipeline: pipeline} + _, err := runner.Run(cmdio.MockDiscard(t.Context()), &Options{Pipeline: *opts, NoWait: true}) + require.NoError(t, err) + require.Equal(t, []string{"`a`` b,c`", "orders"}, opts.Refresh) + }) + } +} + +func TestPipelineRunnerRestartFileSelections(t *testing.T) { + tests := []struct { + name string + continuous bool + full bool + wildcard bool + getErr error + err string + }{ + {name: "continuous refresh", continuous: true, err: "file selectors with --restart require a triggered pipeline"}, + {name: "continuous full refresh", continuous: true, full: true, err: "file selectors with --restart require a triggered pipeline"}, + {name: "triggered refresh"}, + {name: "triggered full refresh", full: true}, + {name: "continuous file selection with wildcard", continuous: true, full: true, wildcard: true, err: "file selectors with --restart require a triggered pipeline"}, + {name: "triggered file selection with wildcard", full: true, wildcard: true}, + {name: "cannot read deployed pipeline", getErr: errors.New("cannot read pipeline"), err: "cannot read pipeline"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pipeline := &resources.Pipeline{ID: "123"} + b := &bundle.Bundle{} + m := mocks.NewMockWorkspaceClient(t) + m.WorkspaceClient.Config = &sdk_config.Config{Host: "https://test.test"} + b.SetWorkpaceClient(m.WorkspaceClient) + api := m.GetMockPipelinesAPI() + api.EXPECT().GetByPipelineId(mock.Anything, "123").Return(&pipelines.GetPipelineResponse{ + Spec: &pipelines.PipelineSpec{Continuous: tt.continuous}, //nolint:staticcheck // Exercise the supported legacy pipeline mode. + }, tt.getErr).Once() + opts := &Options{NoWait: true} + if tt.full { + opts.Pipeline.FullRefreshByPath = []string{"/Workspace/Shared/orders.py"} + } else { + opts.Pipeline.RefreshByPath = []string{"/Workspace/Shared/orders.py"} + } + if tt.wildcard { + opts.Pipeline.Refresh = []string{"*"} + } + if tt.err == "" { + wait := &pipelines.WaitGetPipelineIdle[struct{}]{ + Poll: func(time.Duration, func(*pipelines.GetPipelineResponse)) (*pipelines.GetPipelineResponse, error) { + return nil, nil + }, + } + api.EXPECT().Stop(mock.Anything, pipelines.StopRequest{PipelineId: "123"}).Return(wait, nil).Once() + api.EXPECT().StartUpdate(mock.Anything, pipelines.StartUpdate{ + PipelineId: "123", RefreshSelection: opts.Pipeline.Refresh, + RefreshSelectionByPath: opts.Pipeline.RefreshByPath, FullRefreshSelectionByPath: opts.Pipeline.FullRefreshByPath, + }).Return(&pipelines.StartUpdateResponse{UpdateId: "456"}, nil).Once() + } + runner := pipelineRunner{key: "test", bundle: b, pipeline: pipeline} + _, err := runner.Restart(cmdio.MockDiscard(t.Context()), opts) + if tt.err != "" { + require.ErrorContains(t, err, tt.err) + api.AssertNotCalled(t, "Stop", mock.Anything, mock.Anything) + api.AssertNotCalled(t, "StartUpdate", mock.Anything, mock.Anything) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 17199a6cabf..618109f216f 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -108,6 +108,8 @@ parameter names. If the specified job does not use job parameters and the job has a Python file task or a Python wheel task, the second example applies. +` + run.PipelineRefreshHelp + ` + --------------------------------------------------------- You can also use the bundle run command to execute scripts / commands in the same @@ -206,6 +208,12 @@ Example usage: return err } + if ref.Description.SingularName == "pipeline" { + if err := runOptions.Pipeline.ResolveRefreshSelectors(ctx, b); err != nil { + return err + } + } + runOptions.NoWait = noWait var runOutput output.RunOutput if restart { diff --git a/cmd/pipelines/run.go b/cmd/pipelines/run.go index 9f81961b6f0..cc85f32e6a0 100644 --- a/cmd/pipelines/run.go +++ b/cmd/pipelines/run.go @@ -17,6 +17,7 @@ import ( bundleresources "github.com/databricks/cli/bundle/resources" "github.com/databricks/cli/bundle/run" bundlerunoutput "github.com/databricks/cli/bundle/run/output" + "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdgroup" @@ -179,15 +180,18 @@ func fetchAndDisplayPipelineUpdate(ctx context.Context, w *databricks.WorkspaceC latestUpdate := *getUpdateResponse.Update - params := &PipelineEventsQueryParams{ - Filter: fmt.Sprintf("update_id='%s' AND event_type='update_progress'", updateId), - OrderBy: "timestamp asc", - } - - events, err := fetchAllPipelineEvents(ctx, w, pipelineId, params) + // The public events API rejects event_type filters. + events, err := w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ + PipelineId: pipelineId, + Filter: fmt.Sprintf("update_id='%s'", updateId), + OrderBy: []string{"timestamp asc"}, + }) if err != nil { return err } + events = slices.DeleteFunc(events, func(event pipelines.PipelineEvent) bool { + return event.EventType != "update_progress" + }) err = displayPipelineUpdate(ctx, latestUpdate, pipelineId, events) if err != nil { @@ -233,17 +237,16 @@ func runCommand() *cobra.Command { Long: `Run the pipeline identified by KEY. KEY is the unique name of the pipeline to run, as defined in its YAML file. If there is only one pipeline in the project, KEY is optional and the pipeline will be auto-selected. -Refreshes all tables in the pipeline unless otherwise specified.`, +Refreshes all tables in the pipeline unless otherwise specified. + +` + run.PipelineRefreshHelp, + Args: root.MaximumNArgs(1), } - var refresh []string - var fullRefreshAll bool - var fullRefresh []string + var pipelineOptions run.PipelineOptions pipelineGroup := cmdgroup.NewFlagGroup("Pipeline Run") - pipelineGroup.FlagSet().StringSliceVar(&refresh, "refresh", nil, "List of tables to run.") - pipelineGroup.FlagSet().BoolVar(&fullRefreshAll, "full-refresh-all", false, "Perform a full graph reset and recompute.") - pipelineGroup.FlagSet().StringSliceVar(&fullRefresh, "full-refresh", nil, "List of tables to reset and recompute.") + pipelineOptions.DefineRefreshFlags(pipelineGroup.FlagSet()) wrappedCmd := cmdgroup.NewCommandWithGroupFlag(cmd) wrappedCmd.AddFlagGroup(pipelineGroup) @@ -257,6 +260,10 @@ Refreshes all tables in the pipeline unless otherwise specified.`, ctx := logdiag.InitContext(cmd.Context()) cmd.SetContext(ctx) + runOptions := run.Options{ + Pipeline: pipelineOptions, + NoWait: noWait, + } var key string b, err := utils.ProcessBundle(cmd, utils.ProcessOptions{ PostInitFunc: func(ctx context.Context, b *bundle.Bundle) error { @@ -264,6 +271,9 @@ Refreshes all tables in the pipeline unless otherwise specified.`, key, _, err = resolveRunArgument(ctx, b, args) return err }, + PostStateFunc: func(ctx context.Context, b *bundle.Bundle, _ *statemgmt.StateDesc) error { + return runOptions.Pipeline.ResolveRefreshSelectors(ctx, b) + }, ErrorOnEmptyState: true, SkipInitContext: true, }) @@ -278,15 +288,6 @@ Refreshes all tables in the pipeline unless otherwise specified.`, return err } - runOptions := run.Options{ - Pipeline: run.PipelineOptions{ - Refresh: refresh, - FullRefreshAll: fullRefreshAll, - FullRefresh: fullRefresh, - }, - NoWait: noWait, - } - var runOutput bundlerunoutput.RunOutput if restart { runOutput, err = runner.Restart(ctx, &runOptions) diff --git a/cmd/pipelines/run_test.go b/cmd/pipelines/run_test.go index 9937cb83d0d..246380a8c8e 100644 --- a/cmd/pipelines/run_test.go +++ b/cmd/pipelines/run_test.go @@ -5,10 +5,46 @@ import ( "time" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestFetchAndDisplayPipelineUpdateFiltersProgressAcrossPages(t *testing.T) { + server := testserver.New(t) + server.Handle("GET", "/api/2.0/pipelines/pipeline-1/updates/update-1", func(req testserver.Request) any { + return pipelines.GetUpdateResponse{Update: &pipelines.UpdateInfo{UpdateId: "update-1"}} + }) + server.Handle("GET", "/api/2.0/pipelines/pipeline-1/events", func(req testserver.Request) any { + if req.URL.Query().Get("page_token") == "" { + assert.Equal(t, "update_id='update-1'", req.URL.Query().Get("filter")) + return pipelines.ListPipelineEventsResponse{ + NextPageToken: "next-page", + Events: []pipelines.PipelineEvent{ + {EventType: "update_progress", Timestamp: "2026-01-01T00:00:00Z", Message: "Update is WAITING_FOR_RESOURCES."}, + {EventType: "flow_progress", Timestamp: "2026-01-01T00:00:05Z", Message: "Flow is PLANNING."}, + }, + } + } + assert.Equal(t, "next-page", req.URL.Query().Get("page_token")) + return pipelines.ListPipelineEventsResponse{ + Events: []pipelines.PipelineEvent{ + {EventType: "update_progress", Timestamp: "2026-01-01T00:00:10Z", Message: "Update is COMPLETED."}, + }, + } + }) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "test-token"}) + require.NoError(t, err) + ctx, stdout := cmdio.NewTestContextWithStdout(t.Context()) + + err = fetchAndDisplayPipelineUpdate(ctx, w, "pipeline-1", "update-1") + require.NoError(t, err) + assert.Contains(t, stdout.String(), "WAITING_FOR_RESOURCES") + assert.Contains(t, stdout.String(), "10.0s") +} + func TestDisplayPipelineUpdate(t *testing.T) { tests := []struct { name string @@ -99,6 +135,34 @@ Update for pipeline completed successfully. Pipeline configurations for this update: • Refreshed [table1, table2] • Full refreshed [table3] +`, + }, + { + name: "pipeline update with file refresh selections", + update: pipelines.UpdateInfo{ + RefreshSelectionByPath: []string{"/Workspace/Shared/source.sql"}, + FullRefreshSelectionByPath: []string{"/Workspace/Shared/notebook"}, + }, + expected: ` +Update for pipeline completed successfully. + +Pipeline configurations for this update: +• Refreshed tables defined in [/Workspace/Shared/source.sql] +• Full refreshed tables defined in [/Workspace/Shared/notebook] +`, + }, + { + name: "pipeline update with table refresh and file full refresh", + update: pipelines.UpdateInfo{ + RefreshSelection: []string{"table1"}, + FullRefreshSelectionByPath: []string{"/Workspace/Shared/source.sql"}, + }, + expected: ` +Update for pipeline completed successfully. + +Pipeline configurations for this update: +• Refreshed [table1] +• Full refreshed tables defined in [/Workspace/Shared/source.sql] `, }, { diff --git a/cmd/pipelines/templates.go b/cmd/pipelines/templates.go index 3c471b366bb..f0bc50eedf1 100644 --- a/cmd/pipelines/templates.go +++ b/cmd/pipelines/templates.go @@ -29,7 +29,7 @@ Update end time: {{ .LastEventTime }} Pipeline configurations for this update: {{- if .Update.FullRefresh }} • All tables are fully refreshed -{{- else if and (eq (len .Update.RefreshSelection) 0) (eq (len .Update.FullRefreshSelection) 0) }} +{{- else if and (eq (len .Update.RefreshSelection) 0) (eq (len .Update.FullRefreshSelection) 0) (eq (len .Update.RefreshSelectionByPath) 0) (eq (len .Update.FullRefreshSelectionByPath) 0) }} • All tables are refreshed {{- else }} {{- if gt (len .Update.RefreshSelection) 0 }} @@ -38,6 +38,12 @@ Pipeline configurations for this update: {{- if gt (len .Update.FullRefreshSelection) 0 }} • Full refreshed [{{ join .Update.FullRefreshSelection ", " }}] {{- end }} +{{- if gt (len .Update.RefreshSelectionByPath) 0 }} +• Refreshed tables defined in [{{ join .Update.RefreshSelectionByPath ", " }}] +{{- end }} +{{- if gt (len .Update.FullRefreshSelectionByPath) 0 }} +• Full refreshed tables defined in [{{ join .Update.FullRefreshSelectionByPath ", " }}] +{{- end }} {{- end }} {{- if .Update.Cause }} • Update cause: {{ .Update.Cause }} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 98c52d1fc6c..a719aa911f3 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -222,7 +222,7 @@ type FakeWorkspace struct { JobRunOutputs map[int64]jobs.RunOutput JobRunIdempotency map[string]int64 Pipelines map[string]pipelines.GetPipelineResponse - PipelineUpdates map[string]bool + PipelineUpdates map[string]pipelines.UpdateInfo Monitors map[string]catalog.MonitorInfo DataQualityMonitors map[string]dataquality.Monitor Apps map[string]apps.App @@ -533,7 +533,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { JobRunIdempotency: map[string]int64{}, Grants: map[string][]catalog.PrivilegeAssignment{}, Pipelines: map[string]pipelines.GetPipelineResponse{}, - PipelineUpdates: map[string]bool{}, + PipelineUpdates: map[string]pipelines.UpdateInfo{}, Monitors: map[string]catalog.MonitorInfo{}, DataQualityMonitors: map[string]dataquality.Monitor{}, Apps: map[string]apps.App{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 8d7708af5af..fbd43d17013 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -674,7 +674,7 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("POST", "/api/2.0/pipelines/{pipeline_id}/updates", func(req Request) any { - return req.Workspace.PipelineStartUpdate(req.Vars["pipeline_id"]) + return req.Workspace.PipelineStartUpdate(req, req.Vars["pipeline_id"]) }) server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}/events", func(req Request) any { diff --git a/libs/testserver/pipelines.go b/libs/testserver/pipelines.go index 0aedc44c5d3..6960aebf350 100644 --- a/libs/testserver/pipelines.go +++ b/libs/testserver/pipelines.go @@ -143,7 +143,7 @@ func (s *FakeWorkspace) PipelineUpdate(req Request, pipelineId string) Response return Response{} } -func (s *FakeWorkspace) PipelineStartUpdate(pipelineId string) Response { +func (s *FakeWorkspace) PipelineStartUpdate(req Request, pipelineId string) Response { defer s.LockUnlock()() _, exists := s.Pipelines[pipelineId] @@ -154,8 +154,25 @@ func (s *FakeWorkspace) PipelineStartUpdate(pipelineId string) Response { } } + var start pipelines.StartUpdate + if err := json.Unmarshal(req.Body, &start); err != nil { + return Response{ + Body: fmt.Sprintf("cannot unmarshal request body: %s", err), + StatusCode: 400, + } + } + updateId := nextUUID() - s.PipelineUpdates[updateId] = true + // GetUpdate echoes selectors; the execution engine resolves paths to tables. + s.PipelineUpdates[updateId] = pipelines.UpdateInfo{ + UpdateId: updateId, + State: pipelines.UpdateInfoStateCompleted, + FullRefresh: start.FullRefresh, + RefreshSelection: start.RefreshSelection, + FullRefreshSelection: start.FullRefreshSelection, + RefreshSelectionByPath: start.RefreshSelectionByPath, + FullRefreshSelectionByPath: start.FullRefreshSelectionByPath, + } return Response{ Body: pipelines.StartUpdateResponse{ @@ -194,7 +211,7 @@ func (s *FakeWorkspace) PipelineGetUpdate(pipelineId, updateId string) Response } // Check if the update exists - _, updateExists := s.PipelineUpdates[updateId] + update, updateExists := s.PipelineUpdates[updateId] if !updateExists { return Response{ StatusCode: 404, @@ -204,10 +221,7 @@ func (s *FakeWorkspace) PipelineGetUpdate(pipelineId, updateId string) Response return Response{ Body: pipelines.GetUpdateResponse{ - Update: &pipelines.UpdateInfo{ - UpdateId: updateId, - State: pipelines.UpdateInfoStateCompleted, - }, + Update: &update, }, } } diff --git a/libs/testserver/pipelines_test.go b/libs/testserver/pipelines_test.go index 39114714082..6bb1dadfbdb 100644 --- a/libs/testserver/pipelines_test.go +++ b/libs/testserver/pipelines_test.go @@ -51,7 +51,7 @@ func TestPipelineCreate_AllowsSingleSegmentTargetSchemaName(t *testing.T) { func TestPipelineStartUpdate_HandlesNonExistentPipeline(t *testing.T) { workspace := NewFakeWorkspace("http://test", "dbapi123") - response := workspace.PipelineStartUpdate("non-existent-pipeline") + response := workspace.PipelineStartUpdate(Request{Body: []byte(`{}`)}, "non-existent-pipeline") assert.Equal(t, 404, response.StatusCode) assert.Contains(t, response.Body.(map[string]string)["message"], "The specified pipeline non-existent-pipeline was not found") } @@ -74,7 +74,7 @@ func TestPipelineStop_AfterUpdate(t *testing.T) { pipelineId := createTestPipeline(t, workspace) - startResponse := workspace.PipelineStartUpdate(pipelineId) + startResponse := workspace.PipelineStartUpdate(Request{Body: []byte(`{}`)}, pipelineId) assert.Equal(t, 0, startResponse.StatusCode) stopResponse := workspace.PipelineStop(pipelineId) @@ -85,3 +85,43 @@ func TestPipelineStop_AfterUpdate(t *testing.T) { assert.Equal(t, pipelineId, stopBody.PipelineId) assert.Equal(t, pipelines.PipelineStateIdle, stopBody.State) } + +func TestPipelineGetUpdatePreservesRefreshSelections(t *testing.T) { + workspace := NewFakeWorkspace("http://workspace.test", "dbapi123") + pipelineId := createTestPipeline(t, workspace) + tests := []struct { + name string + body string + want pipelines.UpdateInfo + }{ + { + name: "file selections", + body: `{"refresh_selection_by_path":["/Workspace/src/orders.py","/Workspace/src/returns.py"],"full_refresh_selection_by_path":["/Workspace/src/customers.sql"]}`, + want: pipelines.UpdateInfo{ + RefreshSelectionByPath: []string{"/Workspace/src/orders.py", "/Workspace/src/returns.py"}, + FullRefreshSelectionByPath: []string{"/Workspace/src/customers.sql"}, + }, + }, + { + name: "table selections", + body: `{"refresh_selection":["orders"],"full_refresh_selection":["customers"]}`, + want: pipelines.UpdateInfo{ + RefreshSelection: []string{"orders"}, + FullRefreshSelection: []string{"customers"}, + }, + }, + } + for i := range tests { + response := workspace.PipelineStartUpdate(Request{Body: []byte(tests[i].body)}, pipelineId) + require.Zero(t, response.StatusCode) + tests[i].want.UpdateId = response.Body.(pipelines.StartUpdateResponse).UpdateId + tests[i].want.State = pipelines.UpdateInfoStateCompleted + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + response := workspace.PipelineGetUpdate(pipelineId, tt.want.UpdateId) + require.Zero(t, response.StatusCode) + assert.Equal(t, &tt.want, response.Body.(pipelines.GetUpdateResponse).Update) + }) + } +}