diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index 99a5cd9912f..bf63a23f6ef 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -23,10 +23,16 @@ example: databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow' +The command waits for the results. It exits with code 0 only if at least one +test was collected and none failed or errored, so a run whose tests were all +skipped succeeds; otherwise it exits with code 1. --no-wait returns after the +update starts. To stop the update, run 'pipelines stop'. + Usage: databricks pipelines test [flags] [KEY] [TEST_SELECTION...] Pipeline Test Flags: + --no-wait Don't wait for the tests to complete. --pipeline-id string ID of the pipeline to test, instead of a KEY. --pytest-arg stringArray Argument to pass to pytest (repeatable). Only -k and -m are supported. diff --git a/acceptance/pipelines/test/immutable/output.txt b/acceptance/pipelines/test/immutable/output.txt index 02f1cbaef42..04fcaaaeda7 100644 --- a/acceptance/pipelines/test/immutable/output.txt +++ b/acceptance/pipelines/test/immutable/output.txt @@ -15,7 +15,7 @@ If you've made local changes, run 'databricks pipelines deploy' first to ensure Error: cannot resolve test selection "tests/test_orders.py::test_backfill": local test paths are not supported with workspace.sync_mode immutable; pass an absolute workspace path === Absolute workspace paths and running all tests still work ->>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill +>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -23,7 +23,7 @@ If you've made local changes, run 'databricks pipelines deploy' first to ensure Pipeline: my_pipeline ([UUID][0]) Update: [UUID][1] ->>> [CLI] pipelines test my_pipeline +>>> [CLI] pipelines test my_pipeline --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. diff --git a/acceptance/pipelines/test/immutable/script b/acceptance/pipelines/test/immutable/script index 27025924735..e78d8a60a9c 100644 --- a/acceptance/pipelines/test/immutable/script +++ b/acceptance/pipelines/test/immutable/script @@ -5,6 +5,6 @@ title "Local test paths are not supported with workspace.sync_mode immutable" trace musterr $CLI pipelines test my_pipeline tests/test_orders.py::test_backfill title "Absolute workspace paths and running all tests still work" -trace MSYS_NO_PATHCONV=1 $CLI pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill -trace $CLI pipelines test my_pipeline +trace MSYS_NO_PATHCONV=1 $CLI pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill --no-wait +trace $CLI pipelines test my_pipeline --no-wait trace print_requests.py //updates diff --git a/acceptance/pipelines/test/pipeline-id/output.txt b/acceptance/pipelines/test/pipeline-id/output.txt index 9b40c5ca1fa..46c03562359 100644 --- a/acceptance/pipelines/test/pipeline-id/output.txt +++ b/acceptance/pipelines/test/pipeline-id/output.txt @@ -1,6 +1,6 @@ === Relative test selections resolve against the pipeline root_path ->>> MSYS_NO_PATHCONV=1 [CLI] pipelines test --pipeline-id [UUID][0] tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests +>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test --pipeline-id [UUID][0] tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests --no-wait Pipeline: [UUID][0] Update: [UUID][1] @@ -23,7 +23,7 @@ Update: [UUID][1] >>> musterr [CLI] pipelines test --pipeline-id [UUID][2] tests Error: cannot resolve test selection "tests": the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests ->>> MSYS_NO_PATHCONV=1 [CLI] pipelines test --pipeline-id [UUID][2] /Workspace/Users/someone@example.com/tests/test_other.py::test_x +>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test --pipeline-id [UUID][2] /Workspace/Users/someone@example.com/tests/test_other.py::test_x --no-wait Pipeline: [UUID][2] Update: [UUID][3] diff --git a/acceptance/pipelines/test/pipeline-id/script b/acceptance/pipelines/test/pipeline-id/script index 3745fe70c50..bf8ae479ce7 100644 --- a/acceptance/pipelines/test/pipeline-id/script +++ b/acceptance/pipelines/test/pipeline-id/script @@ -3,12 +3,12 @@ no_root_pipeline_id=$($CLI pipelines create --json '{"name": "no-root"}' | jq -r rm "$OUT_REQUESTS" title "Relative test selections resolve against the pipeline root_path" -trace MSYS_NO_PATHCONV=1 $CLI pipelines test --pipeline-id $pipeline_id tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests +trace MSYS_NO_PATHCONV=1 $CLI pipelines test --pipeline-id $pipeline_id tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests --no-wait trace print_requests.py //updates title "Without a root_path, test selections must be workspace paths" trace musterr $CLI pipelines test --pipeline-id $no_root_pipeline_id tests -trace MSYS_NO_PATHCONV=1 $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Users/someone@example.com/tests/test_other.py::test_x +trace MSYS_NO_PATHCONV=1 $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Users/someone@example.com/tests/test_other.py::test_x --no-wait trace print_requests.py //updates title "Outside a project, --pipeline-id is required" diff --git a/acceptance/pipelines/test/results/failed/out.test.toml b/acceptance/pipelines/test/results/failed/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/results/failed/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/results/failed/output.txt b/acceptance/pipelines/test/results/failed/output.txt new file mode 100644 index 00000000000..72c51649bc1 --- /dev/null +++ b/acceptance/pipelines/test/results/failed/output.txt @@ -0,0 +1,20 @@ + +=== Failed, errored and skipped tests +>>> musterr [CLI] pipelines test --pipeline-id [UUID][0] +Pipeline: [UUID][0] +Update: [UUID][1] +Running tests... +1 of 4 PASSED tests/test_orders.py::test_valid_order ........................... [PASSED] +2 of 4 FAILED tests/test_orders.py::test_backfill .............................. [FAILED in 2.14s] +3 of 4 SKIPPED tests/test_orders.py::test_legacy_format ........................ [SKIPPED] +4 of 4 ERROR tests/test_orders.py::test_fixture ................................ [ERROR] + +Completed with 1 failed and 1 errored tests: + +Failure in tests/test_orders.py::test_backfill + AssertionError: assert 100 == 150 + +Error in tests/test_orders.py::test_fixture + RuntimeError: fixture orders setup failed + +Done. PASSED=1 FAILED=1 SKIPPED=1 ERROR=1 TOTAL=4 diff --git a/acceptance/pipelines/test/results/failed/script b/acceptance/pipelines/test/results/failed/script new file mode 100644 index 00000000000..4942d54e76f --- /dev/null +++ b/acceptance/pipelines/test/results/failed/script @@ -0,0 +1,2 @@ +title "Failed, errored and skipped tests" +trace musterr $CLI pipelines test --pipeline-id $pipeline_id diff --git a/acceptance/pipelines/test/results/failed/test.toml b/acceptance/pipelines/test/results/failed/test.toml new file mode 100644 index 00000000000..54707dfe879 --- /dev/null +++ b/acceptance/pipelines/test/results/failed/test.toml @@ -0,0 +1,25 @@ +[[Server]] +Pattern = "POST /api/2.0/pipelines/{pipeline_id}/updates" +Response.Body = '{"update_id": "00000000-0000-0000-0000-0000000000a1"}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/updates/{update_id}" +Response.Body = '{"update": {"update_id": "00000000-0000-0000-0000-0000000000a1", "state": "FAILED"}}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/events" +Response.Body = ''' +{"events": [ + {"id": "e1", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e2", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e3", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_legacy_format"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e4", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_fixture"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e5", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:01.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order"}, "details": {"test_case_progress": {"status": "PASSED"}}}, + {"id": "e6", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:01.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "RUNNING"}}}, + {"id": "e7", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:03.140Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "FAILED"}}, "level": "ERROR", "error": {"fatal": false, "exceptions": [{"class_name": "AssertionError", "message": "assert 100 == 150"}]}}, + {"id": "e8", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:04.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_legacy_format"}, "details": {"test_case_progress": {"status": "SKIPPED"}}}, + {"id": "e9", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:05.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_fixture"}, "details": {"test_case_progress": {"status": "ERROR"}}, "level": "ERROR", "error": {"fatal": false, "exceptions": [{"class_name": "RuntimeError", "message": "fixture orders setup failed"}]}}, + {"id": "e10", "event_type": "test_run_summary", "timestamp": "2026-01-01T00:00:06.000Z", "details": {"test_run_summary": {"total": 4, "passed": 1, "failed": 1, "skipped": 1, "errored": 1}}}, + {"id": "e11", "event_type": "update_progress", "timestamp": "2026-01-01T00:00:07.000Z", "level": "ERROR", "error": {"fatal": true, "exceptions": [{"class_name": "TestFailure", "message": "[PIPELINE_TEST_FAILED] 2 of the pipeline's tests failed (1 passed, 1 skipped)."}]}} +]} +''' diff --git a/acceptance/pipelines/test/results/incomplete/out.test.toml b/acceptance/pipelines/test/results/incomplete/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/results/incomplete/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/results/incomplete/output.txt b/acceptance/pipelines/test/results/incomplete/output.txt new file mode 100644 index 00000000000..9bc31359298 --- /dev/null +++ b/acceptance/pipelines/test/results/incomplete/output.txt @@ -0,0 +1,11 @@ + +=== Compute ends before all results are received +>>> musterr [CLI] pipelines test --pipeline-id [UUID][0] +Pipeline: [UUID][0] +Update: [UUID][1] +Running tests... +1 of 2 PASSED tests/test_compute.py::test_small_order .......................... [PASSED] + +RUN INCOMPLETE: the update failed before all test results were received. +Done. PASSED=1 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=2 +Error: update [UUID][1] is FAILED: ComputeTerminatedException: Compute terminated unexpectedly. diff --git a/acceptance/pipelines/test/results/incomplete/script b/acceptance/pipelines/test/results/incomplete/script new file mode 100644 index 00000000000..5a3bdb6836d --- /dev/null +++ b/acceptance/pipelines/test/results/incomplete/script @@ -0,0 +1,2 @@ +title "Compute ends before all results are received" +trace musterr $CLI pipelines test --pipeline-id $pipeline_id diff --git a/acceptance/pipelines/test/results/incomplete/test.toml b/acceptance/pipelines/test/results/incomplete/test.toml new file mode 100644 index 00000000000..d78f4db704d --- /dev/null +++ b/acceptance/pipelines/test/results/incomplete/test.toml @@ -0,0 +1,19 @@ +[[Server]] +Pattern = "POST /api/2.0/pipelines/{pipeline_id}/updates" +Response.Body = '{"update_id": "00000000-0000-0000-0000-0000000000a1"}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/updates/{update_id}" +Response.Body = '{"update": {"update_id": "00000000-0000-0000-0000-0000000000a1", "state": "FAILED"}}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/events" +Response.Body = ''' +{"events": [ + {"id": "e1", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_compute.py::test_small_order"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e2", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_compute.py::test_large_order"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e3", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:01.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_compute.py::test_small_order"}, "details": {"test_case_progress": {"status": "PASSED"}}}, + {"id": "e4", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:01.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_compute.py::test_large_order"}, "details": {"test_case_progress": {"status": "RUNNING"}}}, + {"id": "e5", "event_type": "update_progress", "timestamp": "2026-01-01T00:00:02.000Z", "level": "ERROR", "error": {"fatal": true, "exceptions": [{"class_name": "ComputeTerminatedException", "message": "Compute terminated unexpectedly."}]}} +]} +''' diff --git a/acceptance/pipelines/test/results/passed/out.test.toml b/acceptance/pipelines/test/results/passed/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/results/passed/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/results/passed/output.txt b/acceptance/pipelines/test/results/passed/output.txt new file mode 100644 index 00000000000..2db22a1cd0a --- /dev/null +++ b/acceptance/pipelines/test/results/passed/output.txt @@ -0,0 +1,30 @@ + +=== All tests pass +>>> [CLI] pipelines test --pipeline-id [UUID][0] +Pipeline: [UUID][0] +Update: [UUID][1] +Running tests... +1 of 2 PASSED tests/test_orders.py::test_valid_order ........................... [PASSED in 1.84s] +2 of 2 PASSED tests/test_orders.py::test_backfill .............................. [PASSED in 1.00s] + +Done. PASSED=2 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=2 + +>>> print_requests.py --get //events +{ + "method": "GET", + "path": "/api/2.0/pipelines/[UUID][0]/events", + "q": { + "filter": "update_id = '[UUID][1]'", + "max_results": "250", + "order_by": "timestamp asc" + } +} +{ + "method": "GET", + "path": "/api/2.0/pipelines/[UUID][0]/events", + "q": { + "filter": "update_id = '[UUID][1]'", + "max_results": "250", + "order_by": "timestamp asc" + } +} diff --git a/acceptance/pipelines/test/results/passed/script b/acceptance/pipelines/test/results/passed/script new file mode 100644 index 00000000000..2d87eb30ab9 --- /dev/null +++ b/acceptance/pipelines/test/results/passed/script @@ -0,0 +1,3 @@ +title "All tests pass" +trace $CLI pipelines test --pipeline-id $pipeline_id +trace print_requests.py --get //events diff --git a/acceptance/pipelines/test/results/passed/test.toml b/acceptance/pipelines/test/results/passed/test.toml new file mode 100644 index 00000000000..f821ca318eb --- /dev/null +++ b/acceptance/pipelines/test/results/passed/test.toml @@ -0,0 +1,21 @@ +[[Server]] +Pattern = "POST /api/2.0/pipelines/{pipeline_id}/updates" +Response.Body = '{"update_id": "00000000-0000-0000-0000-0000000000a1"}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/updates/{update_id}" +Response.Body = '{"update": {"update_id": "00000000-0000-0000-0000-0000000000a1", "state": "COMPLETED"}}' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/events" +Response.Body = ''' +{"events": [ + {"id": "e1", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e2", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "PENDING"}}}, + {"id": "e3", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:01.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order"}, "details": {"test_case_progress": {"status": "RUNNING"}}}, + {"id": "e4", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:02.840Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order"}, "details": {"test_case_progress": {"status": "PASSED"}}}, + {"id": "e5", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:03.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "RUNNING"}}}, + {"id": "e6", "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:04.000Z", "origin": {"test_case_id": "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill"}, "details": {"test_case_progress": {"status": "PASSED"}}}, + {"id": "e7", "event_type": "test_run_summary", "timestamp": "2026-01-01T00:00:05.000Z", "details": {"test_run_summary": {"total": 2, "passed": 2, "failed": 0, "skipped": 0, "errored": 0}}} +]} +''' diff --git a/acceptance/pipelines/test/results/script.prepare b/acceptance/pipelines/test/results/script.prepare new file mode 100644 index 00000000000..caaa90ba318 --- /dev/null +++ b/acceptance/pipelines/test/results/script.prepare @@ -0,0 +1,3 @@ +# Each case stubs Start Update, Get Update and List Pipeline Events in its test.toml. +pipeline_id=$($CLI pipelines create --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id) +rm "$OUT_REQUESTS" diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt index 9a9e80db6e6..ae6afcb4754 100644 --- a/acceptance/pipelines/test/selection/output.txt +++ b/acceptance/pipelines/test/selection/output.txt @@ -7,7 +7,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID] === Run all tests of the only pipeline ->>> [CLI] pipelines test +>>> [CLI] pipelines test --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -25,7 +25,7 @@ Update: [UUID][1] } === Select tests by relative path and test ID, with pytest arguments in both --pytest-arg forms ->>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg -m --pytest-arg smoke,fast +>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg -m --pytest-arg smoke,fast --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -56,7 +56,7 @@ Update: [UUID][2] } === Relative paths resolve from the working directory ->>> [CLI] pipelines test my_pipeline test_orders.py::test_valid_order +>>> [CLI] pipelines test my_pipeline test_orders.py::test_valid_order --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. diff --git a/acceptance/pipelines/test/selection/script b/acceptance/pipelines/test/selection/script index 5e7efd415be..3344ef8a37d 100644 --- a/acceptance/pipelines/test/selection/script +++ b/acceptance/pipelines/test/selection/script @@ -2,16 +2,16 @@ trace $CLI pipelines deploy rm "$OUT_REQUESTS" title "Run all tests of the only pipeline" -trace $CLI pipelines test +trace $CLI pipelines test --no-wait trace print_requests.py //updates title "Select tests by relative path and test ID, with pytest arguments in both --pytest-arg forms" -trace MSYS_NO_PATHCONV=1 $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg -m --pytest-arg smoke,fast +trace MSYS_NO_PATHCONV=1 $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg -m --pytest-arg smoke,fast --no-wait trace print_requests.py //updates title "Relative paths resolve from the working directory" cd tests -trace $CLI pipelines test my_pipeline test_orders.py::test_valid_order +trace $CLI pipelines test my_pipeline test_orders.py::test_valid_order --no-wait cd .. trace print_requests.py //updates diff --git a/acceptance/pipelines/test/submit-error/output.txt b/acceptance/pipelines/test/submit-error/output.txt index fc3de068652..52690c65e82 100644 --- a/acceptance/pipelines/test/submit-error/output.txt +++ b/acceptance/pipelines/test/submit-error/output.txt @@ -1,7 +1,7 @@ === A submit error from the service is shown as is, and no update ID is printed >>> musterr [CLI] pipelines test --pipeline-id [UUID][0] -Error: An active update '[UUID][1]' already exists for pipeline '[UUID][0]'. +Error: An active update '[UUID][1]' already exists for pipeline '[UUID][2]'. >>> print_requests.py //updates { diff --git a/acceptance/pipelines/test/submit-error/script b/acceptance/pipelines/test/submit-error/script index 0d0843028c6..405aa9d1473 100644 --- a/acceptance/pipelines/test/submit-error/script +++ b/acceptance/pipelines/test/submit-error/script @@ -1,3 +1,6 @@ +pipeline_id=$($CLI pipelines create --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id) +rm "$OUT_REQUESTS" + title "A submit error from the service is shown as is, and no update ID is printed" -trace musterr $CLI pipelines test --pipeline-id 11111111-2222-3333-4444-555555555555 +trace musterr $CLI pipelines test --pipeline-id $pipeline_id trace print_requests.py //updates diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index dbbc9e042e8..cd3048f5756 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -13,6 +13,7 @@ import ( "path/filepath" "slices" "strings" + "time" "uuid" "github.com/databricks/cli/bundle" @@ -192,6 +193,20 @@ func rootPathSelection(rootPath, relPath string) (string, error) { return strings.TrimSuffix(rootPath, "/") + "/" + filepath.ToSlash(relPath), nil } +// relativeTestID shows a test ID whose file is under folder relative to that folder, as pytest names +// tests from the project root. Other IDs are returned unchanged. The service reports IDs with the +// /Workspace prefix, which a folder may lack, so both are compared without it. +func relativeTestID(folder, id string) string { + if folder == "" { + return id + } + prefix := strings.TrimSuffix(strings.TrimPrefix(folder, "/Workspace"), "/") + "/" + if rel, ok := strings.CutPrefix(strings.TrimPrefix(id, "/Workspace"), prefix); ok { + return rel + } + return id +} + // getRootPath reads the root_path of the deployed pipeline. func getRootPath(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string) (string, error) { pipeline, err := w.Pipelines.Get(ctx, pipelines.GetPipelineRequest{PipelineId: pipelineID}) @@ -233,6 +248,14 @@ type testTarget struct { pipelineID string selections []string w *databricks.WorkspaceClient + + // projectFolder is the deployed folder that test IDs are shown relative to: the bundle's + // synchronized files, or root_path. It is "" when unknown. + projectFolder string +} + +func (t testTarget) shortTestID(id string) string { + return relativeTestID(t.projectFolder, id) } func (t testTarget) printHeader(out io.Writer, updateID string) { @@ -245,7 +268,7 @@ func (t testTarget) printHeader(out io.Writer, updateID string) { } // resolveTestTargetByID identifies the pipeline by --pipeline-id; all positional arguments are selections. -func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) (testTarget, error) { +func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string, wait bool) (testTarget, error) { if cmd.Flag("target").Value.String() == "" { cmd.SetContext(root.SkipLoadBundle(cmd.Context())) } @@ -255,8 +278,9 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) + // root_path resolves relative selections and shortens the test IDs printed while waiting. var rootPath string - if hasRelativeSelection(args) { + if wait || hasRelativeSelection(args) { var err error rootPath, err = getRootPath(ctx, w, pipelineID) if err != nil { @@ -269,7 +293,7 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) if err != nil { return testTarget{}, err } - return testTarget{pipelineID: pipelineID, selections: selections, w: w}, nil + return testTarget{pipelineID: pipelineID, selections: selections, w: w, projectFolder: rootPath}, nil } // resolveTestTargetInBundle identifies the pipeline by its bundle KEY, which must precede any selections. @@ -313,7 +337,97 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, e if err != nil { return testTarget{}, err } - return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx)}, nil + return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx), projectFolder: filesRoot}, nil +} + +const testPollInterval = time.Second + +// waitForTestUpdate polls the update until it ends, prints each test case as it finishes, and +// returns the final state. It takes the steps of the bundle pipeline runner's wait loop +// (bundle/run/pipeline.go), which cannot be reused because it starts a regular update. +func waitForTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipelineID, updateID string, run *testRun, printer *testProgressPrinter) (pipelines.UpdateInfoState, error) { + lastTimestamp := "" + for { + response, err := w.Pipelines.GetUpdateByPipelineIdAndUpdateId(ctx, pipelineID, updateID) + if err != nil { + return "", err + } + if response.Update == nil { + return "", fmt.Errorf("no update found with id %s for pipeline %s", updateID, pipelineID) + } + state := response.Update.State + printer.updateState(state) + + // Reading from the last timestamp returns its events again; the run ignores duplicates. + events, err := readTestEvents(ctx, w, pipelineID, updateID, lastTimestamp) + if err != nil { + return "", err + } + if len(events) > 0 { + lastTimestamp = events[len(events)-1].Timestamp + } + addTestEvents(run, printer, events) + + switch state { + case pipelines.UpdateInfoStateCompleted, pipelines.UpdateInfoStateFailed, pipelines.UpdateInfoStateCanceled: + // Events can become readable after later ones, so read all of them once more. + events, err := readTestEvents(ctx, w, pipelineID, updateID, "") + if err != nil { + return "", err + } + addTestEvents(run, printer, events) + return state, nil + default: + } + + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(testPollInterval): + } + } +} + +// addTestEvents adds events to the run and prints each case that gets its final outcome. +func addTestEvents(run *testRun, printer *testProgressPrinter, events []testPipelineEvent) { + for _, e := range events { + if c := run.add(e); c != nil { + printer.caseFinished(c, len(run.cases)) + } + } +} + +// updateError returns the error that the update reported: its own, or else the test verdict. +func (r *testRun) updateError() string { + if r.fatalError != "" { + return r.fatalError + } + return r.verdictError +} + +// testRunError returns the error the command exits with after the update ended, or nil if at +// least one test was collected and every collected test passed or was skipped. +func testRunError(updateID string, state pipelines.UpdateInfoState, run *testRun, incompleteReason string) error { + if incompleteReason == "" { + summary := run.summaries[0] + testsFailed := summary.Total == 0 || summary.Failed > 0 || summary.Errored > 0 + if !testsFailed && state == pipelines.UpdateInfoStateCompleted && run.fatalError == "" { + return nil + } + // A test update fails when its tests fail; the printed result explains it, unless the + // update ended in another way or reports an error of its own. The service's own verdict, + // such as "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed", is not. + ended := state == pipelines.UpdateInfoStateFailed || state == pipelines.UpdateInfoStateCompleted + if testsFailed && ended && run.fatalError == "" { + return root.ErrAlreadyPrinted + } + } else if state == pipelines.UpdateInfoStateCompleted { + return root.ErrAlreadyPrinted + } + if updateError := run.updateError(); updateError != "" { + return fmt.Errorf("update %s is %s: %s", updateID, state, updateError) + } + return fmt.Errorf("update %s is %s", updateID, state) } // notPipelineKeyError explains that the first argument is not a pipeline KEY, names the bundle's @@ -355,15 +469,22 @@ Each --pytest-arg passes one argument to pytest, in the order given. Only -k and -m are supported, each followed by its value in the next --pytest-arg, for example: - databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow'`, + databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow' + +The command waits for the results. It exits with code 0 only if at least one +test was collected and none failed or errored, so a run whose tests were all +skipped succeeds; otherwise it exits with code 1. --no-wait returns after the +update starts. To stop the update, run 'pipelines stop'.`, } var pipelineID string var pytestArgs []string + var noWait bool testGroup := cmdgroup.NewFlagGroup("Pipeline Test") testGroup.FlagSet().StringVar(&pipelineID, "pipeline-id", "", "ID of the pipeline to test, instead of a KEY.") testGroup.FlagSet().StringArrayVar(&pytestArgs, "pytest-arg", nil, "Argument to pass to pytest (repeatable). Only -k and -m are supported.") + testGroup.FlagSet().BoolVar(&noWait, "no-wait", false, "Don't wait for the tests to complete.") wrappedCmd := cmdgroup.NewCommandWithGroupFlag(cmd) wrappedCmd.AddFlagGroup(testGroup) @@ -389,7 +510,7 @@ example: if err != nil { return fmt.Errorf("--pipeline-id %q is not a pipeline ID", pipelineID) } - target, err = resolveTestTargetByID(cmd, args, id.String()) + target, err = resolveTestTargetByID(cmd, args, id.String(), !noWait) if err != nil { return err } @@ -408,8 +529,22 @@ example: if err != nil { return err } - target.printHeader(cmd.OutOrStdout(), updateID) - return nil + out := cmd.OutOrStdout() + target.printHeader(out, updateID) + if noWait { + return nil + } + + run := newTestRun() + printer := &testProgressPrinter{out: out, shortID: target.shortTestID} + state, err := waitForTestUpdate(ctx, target.w, target.pipelineID, updateID, run, printer) + if err != nil { + return err + } + incompleteReason := run.incompleteReason(state) + renderTestResult(out, run, incompleteReason, target.shortTestID) + renderTestSummaryLine(out, run, incompleteReason) + return testRunError(updateID, state, run, incompleteReason) } return cmd diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index 682567d5b06..7d160a50ec5 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -1,14 +1,20 @@ package pipelines import ( + "bytes" "errors" "os" "path/filepath" + "strings" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/testserver" + databricks "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" @@ -125,3 +131,165 @@ func TestNotPipelineKeyError(t *testing.T) { err = notPipelineKeyError(cmd, &bundle.Bundle{}, "tests", []string{"tests"}) assert.EqualError(t, err, `"tests" is not a pipeline in this bundle, and the bundle has no pipelines`) } + +func TestTestRunError(t *testing.T) { + ev := newTestEvents(t) + tests := []struct { + name string + events func() []testPipelineEvent + state pipelines.UpdateInfoState + wantErr string + }{ + { + name: "incomplete after completion", + events: func() []testPipelineEvent { + return []testPipelineEvent{ev.testCase("t::a", "PASSED", "")} + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: root.ErrAlreadyPrinted.Error(), + }, + { + name: "update failed with the service's verdict although no test failed", + events: func() []testPipelineEvent { + return []testPipelineEvent{ + ev.testCase("t::a", "PASSED", ""), + ev.summary(1, 1, 0, 0, 0), + decodeTestEvent(t, `{"id": "fatal", "error": {"fatal": true, "exceptions": [{"message": "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed."}]}}`), + } + }, + state: pipelines.UpdateInfoStateFailed, + wantErr: "update u1 is FAILED: [PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed.", + }, + { + name: "tests failed although the update completed", + events: func() []testPipelineEvent { + return []testPipelineEvent{ev.testCase("t::a", "FAILED", ""), ev.summary(1, 0, 1, 0, 0)} + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: root.ErrAlreadyPrinted.Error(), + }, + { + name: "no tests collected although the update completed", + events: func() []testPipelineEvent { + return []testPipelineEvent{ev.summary(0, 0, 0, 0, 0)} + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: root.ErrAlreadyPrinted.Error(), + }, + { + name: "all tests passed but the completed update reported a fatal error", + events: func() []testPipelineEvent { + return []testPipelineEvent{ + ev.testCase("t::a", "PASSED", ""), + ev.summary(1, 1, 0, 0, 0), + decodeTestEvent(t, `{"id": "fatal", "error": {"fatal": true, "exceptions": [{"message": "result delivery failed"}]}}`), + } + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: "update u1 is COMPLETED: result delivery failed", + }, + { + name: "all tests were skipped", + events: func() []testPipelineEvent { + return []testPipelineEvent{ev.testCase("t::a", "SKIPPED", ""), ev.summary(1, 0, 0, 1, 0)} + }, + state: pipelines.UpdateInfoStateCompleted, + }, + { + name: "canceled after tests failed", + events: func() []testPipelineEvent { + return []testPipelineEvent{ev.testCase("t::a", "FAILED", ""), ev.summary(1, 0, 1, 0, 0)} + }, + state: pipelines.UpdateInfoStateCanceled, + wantErr: "update u1 is CANCELED", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + run := newTestRun() + for _, e := range tc.events() { + run.add(e) + } + err := testRunError("u1", tc.state, run, run.incompleteReason(tc.state)) + if tc.wantErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.wantErr) + } + }) + } +} + +func TestRelativeTestID(t *testing.T) { + id := "/Workspace/Users/someone/orders/tests/test_a.py::test_x" + assert.Equal(t, "tests/test_a.py::test_x", relativeTestID("/Workspace/Users/someone/orders", id)) + // root_path may lack the /Workspace prefix that the service adds to test IDs. + assert.Equal(t, "tests/test_a.py::test_x", relativeTestID("/Users/someone/orders/", id)) + assert.Equal(t, id, relativeTestID("/Workspace/Users/someone/other", id)) + assert.Equal(t, id, relativeTestID("", id)) +} + +// The wait reads events from the last timestamp, where a later event can share that timestamp, +// and reads all events again after the update ends, where an event can appear late. +func TestWaitForTestUpdate(t *testing.T) { + event := func(id, caseID, status, timestamp string) map[string]any { + return map[string]any{ + "id": id, "event_type": "test_case_progress", "timestamp": timestamp, + "origin": map[string]any{"test_case_id": caseID}, "details": map[string]any{"test_case_progress": map[string]any{"status": status}}, + } + } + a, b := "/Workspace/p/tests/test_a.py::test_a", "/Workspace/p/tests/test_a.py::test_b" + first := []map[string]any{ + event("e1", a, "PENDING", "2026-01-01T00:00:01.000Z"), + event("e2", b, "PENDING", "2026-01-01T00:00:01.000Z"), + event("e3", a, "PASSED", "2026-01-01T00:00:02.000Z"), + } + // e4 has the timestamp of the cursor; e5 and the summary are readable only after the update ends. + tie := append(first[2:], event("e4", b, "RUNNING", "2026-01-01T00:00:02.000Z")) + late := []map[string]any{event("e5", b, "PASSED", "2026-01-01T00:00:03.000Z"), { + "id": "e6", "event_type": "test_run_summary", "timestamp": "2026-01-01T00:00:04.000Z", + "details": map[string]any{"test_run_summary": map[string]any{"total": 2, "passed": 2}}, + }} + + server := testserver.New(t) + polls := 0 + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}/updates/{update_id}", func(req testserver.Request) any { + polls++ + state := "RUNNING" + if polls == 2 { + state = "COMPLETED" + } + return map[string]any{"update": map[string]any{"update_id": "u1", "state": state}} + }) + var filters []string + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}/events", func(req testserver.Request) any { + filter := req.URL.Query().Get("filter") + filters = append(filters, filter) + switch { + case !strings.Contains(filter, "timestamp") && polls == 1: + return map[string]any{"events": first} + case strings.Contains(filter, "timestamp"): + return map[string]any{"events": tie} + default: + return map[string]any{"events": append(append(first, tie...), late...)} + } + }) + + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + var out bytes.Buffer + run := newTestRun() + printer := &testProgressPrinter{out: &out, shortID: func(id string) string { return relativeTestID("/Workspace/p", id) }} + state, err := waitForTestUpdate(t.Context(), w, "p1", "u1", run, printer) + require.NoError(t, err) + + assert.Equal(t, pipelines.UpdateInfoStateCompleted, state) + assert.Equal(t, []string{ + "update_id = 'u1'", + "update_id = 'u1' AND timestamp >= '2026-01-01T00:00:02.000Z'", + "update_id = 'u1'", + }, filters) + assert.Empty(t, run.incompleteReason(state)) + assert.Equal(t, testRunSummary{Total: 2, Passed: 2}, run.counts()) + assert.Contains(t, out.String(), "2 of 2 PASSED tests/test_a.py::test_b") +}