diff --git a/acceptance/pipelines/databricks-cli-help/output.txt b/acceptance/pipelines/databricks-cli-help/output.txt index 8514fa33b2b..adc729f4f5c 100644 --- a/acceptance/pipelines/databricks-cli-help/output.txt +++ b/acceptance/pipelines/databricks-cli-help/output.txt @@ -30,6 +30,7 @@ Main Commands run Run a pipeline show Preview a table's columns and sample rows stop Stop a pipeline + test *Preview* Run the tests of a pipeline Management Commands apply-environment *Public Preview* Apply the latest environment to the pipeline. diff --git a/acceptance/pipelines/test/arguments/out.test.toml b/acceptance/pipelines/test/arguments/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/arguments/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt new file mode 100644 index 00000000000..99a5cd9912f --- /dev/null +++ b/acceptance/pipelines/test/arguments/output.txt @@ -0,0 +1,67 @@ + +>>> [CLI] pipelines test --help +Run the tests of a pipeline in a test-only update. + +This command is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it. + +KEY is the unique name of the pipeline, as defined in its YAML file. +It is required when test selections are given. If there is only one pipeline +in the project and no arguments are given, it is selected automatically. +Outside a project, identify the pipeline with --pipeline-id; all positional +arguments are then test selections. + +TEST_SELECTION is a test directory, file, or pytest test ID, for example +tests/test_orders.py::test_backfill. In a project, a relative path is a local +path, relative to the working directory, and is converted to its deployed +workspace path. With --pipeline-id, a relative path is relative to the +pipeline's root_path. Paths that start with / are workspace paths. Without a +selection, all of the pipeline's tests run. + +Each --pytest-arg passes one argument to pytest, in the order given. Only -k +and -m are supported, each followed by its value in the next --pytest-arg, for +example: + + databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow' + +Usage: + databricks pipelines test [flags] [KEY] [TEST_SELECTION...] + +Pipeline Test Flags: + --pipeline-id string ID of the pipeline to test, instead of a KEY. + --pytest-arg stringArray Argument to pass to pytest (repeatable). Only -k and -m are supported. + +Flags: + -h, --help help for test + +Global Flags: + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in project config. Example: --var="foo=bar" + +=== The command is experimental and off by default +>>> musterr [CLI] pipelines test --pipeline-id 1234 +Error: pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it + +=== pytest arguments must be passed with --pytest-arg +>>> musterr [CLI] pipelines test --pipeline-id 1234 -k orders +Error: -k is a pytest argument: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' + +>>> musterr [CLI] pipelines test --pipeline-id 1234 -- -k orders +Error: the -- separator is not supported: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' + +=== Invalid --pytest-arg values fail before any request +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-x +Error: unsupported pytest argument "-x": only -k and -m are supported + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg= +Error: --pytest-arg requires a non-empty value + +=== The pipeline ID must be a pipeline ID +>>> musterr [CLI] pipelines test --pipeline-id 1234 +Error: --pipeline-id "1234" is not a pipeline ID + +=== JSON output is not supported +>>> musterr [CLI] pipelines test --pipeline-id 1234 -o json +Error: --output json is not supported by pipelines test diff --git a/acceptance/pipelines/test/arguments/script b/acceptance/pipelines/test/arguments/script new file mode 100644 index 00000000000..0df8c20ce2d --- /dev/null +++ b/acceptance/pipelines/test/arguments/script @@ -0,0 +1,18 @@ +trace $CLI pipelines test --help + +title "The command is experimental and off by default" +(unset DATABRICKS_CLI_PIPELINES_TEST; trace musterr $CLI pipelines test --pipeline-id 1234) + +title "pytest arguments must be passed with --pytest-arg" +trace musterr $CLI pipelines test --pipeline-id 1234 -k orders +trace musterr $CLI pipelines test --pipeline-id 1234 -- -k orders + +title "Invalid --pytest-arg values fail before any request" +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-x +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg="" + +title "The pipeline ID must be a pipeline ID" +trace musterr $CLI pipelines test --pipeline-id 1234 + +title "JSON output is not supported" +trace musterr $CLI pipelines test --pipeline-id 1234 -o json diff --git a/acceptance/pipelines/test/immutable/databricks.yml b/acceptance/pipelines/test/immutable/databricks.yml new file mode 100644 index 00000000000..483234d0dd4 --- /dev/null +++ b/acceptance/pipelines/test/immutable/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: test-pipeline-test-immutable + +workspace: + sync_mode: immutable + +resources: + pipelines: + my_pipeline: + name: test-pipeline-test-immutable + libraries: + - file: + path: pipeline_file.py diff --git a/acceptance/pipelines/test/immutable/out.test.toml b/acceptance/pipelines/test/immutable/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/immutable/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/immutable/output.txt b/acceptance/pipelines/test/immutable/output.txt new file mode 100644 index 00000000000..02f1cbaef42 --- /dev/null +++ b/acceptance/pipelines/test/immutable/output.txt @@ -0,0 +1,53 @@ + +>>> [CLI] pipelines deploy +Created internal_immutable_snapshots.immutable +Created pipelines.my_pipeline +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID] + +=== Local test paths are not supported with workspace.sync_mode immutable +>>> musterr [CLI] pipelines test my_pipeline tests/test_orders.py::test_backfill +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: cannot resolve test selection "tests/test_orders.py::test_backfill": local test paths are not supported with workspace.sync_mode immutable; pass an absolute workspace path + +=== Absolute workspace paths and running all tests still work +>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID][0]) +Update: [UUID][1] + +>>> [CLI] pipelines test my_pipeline +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID][0]) +Update: [UUID][2] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID][0]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill" + ] + } + } +} +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID][0]/updates", + "body": { + "test_only": true + } +} diff --git a/acceptance/pipelines/test/immutable/pipeline_file.py b/acceptance/pipelines/test/immutable/pipeline_file.py new file mode 100644 index 00000000000..4a7a3a1878e --- /dev/null +++ b/acceptance/pipelines/test/immutable/pipeline_file.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.table +def orders(): + return spark.range(10) diff --git a/acceptance/pipelines/test/immutable/script b/acceptance/pipelines/test/immutable/script new file mode 100644 index 00000000000..27025924735 --- /dev/null +++ b/acceptance/pipelines/test/immutable/script @@ -0,0 +1,10 @@ +trace $CLI pipelines deploy +rm "$OUT_REQUESTS" + +title "Local test paths are not supported with workspace.sync_mode immutable" +trace musterr $CLI pipelines test my_pipeline tests/test_orders.py::test_backfill + +title "Absolute workspace paths and running all tests still work" +trace MSYS_NO_PATHCONV=1 $CLI pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill +trace $CLI pipelines test my_pipeline +trace print_requests.py //updates diff --git a/acceptance/pipelines/test/immutable/tests/test_orders.py b/acceptance/pipelines/test/immutable/tests/test_orders.py new file mode 100644 index 00000000000..ec31eaa1b01 --- /dev/null +++ b/acceptance/pipelines/test/immutable/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_backfill(): + assert True diff --git a/acceptance/pipelines/test/pipeline-id/out.test.toml b/acceptance/pipelines/test/pipeline-id/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/pipeline-id/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/pipeline-id/output.txt b/acceptance/pipelines/test/pipeline-id/output.txt new file mode 100644 index 00000000000..9b40c5ca1fa --- /dev/null +++ b/acceptance/pipelines/test/pipeline-id/output.txt @@ -0,0 +1,50 @@ + +=== Relative test selections resolve against the pipeline root_path +>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test --pipeline-id [UUID][0] tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests +Pipeline: [UUID][0] +Update: [UUID][1] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID][0]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill", + "/Workspace/Users/someone@example.com/other/tests" + ] + } + } +} + +=== Without a root_path, test selections must be workspace paths +>>> musterr [CLI] pipelines test --pipeline-id [UUID][2] tests +Error: cannot resolve test selection "tests": the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests + +>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test --pipeline-id [UUID][2] /Workspace/Users/someone@example.com/tests/test_other.py::test_x +Pipeline: [UUID][2] +Update: [UUID][3] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID][2]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Users/someone@example.com/tests/test_other.py::test_x" + ] + } + } +} + +=== Outside a project, --pipeline-id is required +>>> musterr [CLI] pipelines test tests +Error: Unable to locate the bundle root: databricks.yml not found. + +Run this command from a directory that contains a bundle, or set the DATABRICKS_BUNDLE_ROOT environment variable to the bundle root directory. +Alternatively, create a new bundle with 'databricks bundle init'. + diff --git a/acceptance/pipelines/test/pipeline-id/script b/acceptance/pipelines/test/pipeline-id/script new file mode 100644 index 00000000000..3745fe70c50 --- /dev/null +++ b/acceptance/pipelines/test/pipeline-id/script @@ -0,0 +1,15 @@ +pipeline_id=$($CLI pipelines create --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id) +no_root_pipeline_id=$($CLI pipelines create --json '{"name": "no-root"}' | jq -r .pipeline_id) +rm "$OUT_REQUESTS" + +title "Relative test selections resolve against the pipeline root_path" +trace MSYS_NO_PATHCONV=1 $CLI pipelines test --pipeline-id $pipeline_id tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests +trace print_requests.py //updates + +title "Without a root_path, test selections must be workspace paths" +trace musterr $CLI pipelines test --pipeline-id $no_root_pipeline_id tests +trace MSYS_NO_PATHCONV=1 $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Users/someone@example.com/tests/test_other.py::test_x +trace print_requests.py //updates + +title "Outside a project, --pipeline-id is required" +trace musterr $CLI pipelines test tests diff --git a/acceptance/pipelines/test/selection/databricks.yml b/acceptance/pipelines/test/selection/databricks.yml new file mode 100644 index 00000000000..0cf849a7919 --- /dev/null +++ b/acceptance/pipelines/test/selection/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: test-pipeline-test-selection + +resources: + pipelines: + my_pipeline: + name: test-pipeline-test-selection + libraries: + - file: + path: pipeline_file.py diff --git a/acceptance/pipelines/test/selection/out.test.toml b/acceptance/pipelines/test/selection/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/selection/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt new file mode 100644 index 00000000000..9a9e80db6e6 --- /dev/null +++ b/acceptance/pipelines/test/selection/output.txt @@ -0,0 +1,83 @@ + +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files... +Created pipelines.my_pipeline +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID] + +=== Run all tests of the only pipeline +>>> [CLI] pipelines test +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID][0]) +Update: [UUID][1] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID][0]/updates", + "body": { + "test_only": true + } +} + +=== Select tests by relative path and test ID, with pytest arguments in both --pytest-arg forms +>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg -m --pytest-arg smoke,fast +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID][0]) +Update: [UUID][2] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID][0]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests", + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests/test_orders.py::test_backfill", + "/Workspace/Users/someone@example.com/tests/test_other.py" + ], + "pytest_args": [ + "-k", + "orders and not slow", + "-m", + "smoke,fast" + ] + } + } +} + +=== Relative paths resolve from the working directory +>>> [CLI] pipelines test my_pipeline test_orders.py::test_valid_order +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID][0]) +Update: [UUID][3] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID][0]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests/test_orders.py::test_valid_order" + ] + } + } +} + +=== Test selections require the pipeline KEY first +>>> musterr [CLI] pipelines test tests/test_orders.py +Error: "tests/test_orders.py" is not a pipeline in this bundle (pipelines: my_pipeline). Usage: databricks pipelines test my_pipeline tests/test_orders.py diff --git a/acceptance/pipelines/test/selection/pipeline_file.py b/acceptance/pipelines/test/selection/pipeline_file.py new file mode 100644 index 00000000000..4a7a3a1878e --- /dev/null +++ b/acceptance/pipelines/test/selection/pipeline_file.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.table +def orders(): + return spark.range(10) diff --git a/acceptance/pipelines/test/selection/script b/acceptance/pipelines/test/selection/script new file mode 100644 index 00000000000..5e7efd415be --- /dev/null +++ b/acceptance/pipelines/test/selection/script @@ -0,0 +1,19 @@ +trace $CLI pipelines deploy +rm "$OUT_REQUESTS" + +title "Run all tests of the only pipeline" +trace $CLI pipelines test +trace print_requests.py //updates + +title "Select tests by relative path and test ID, with pytest arguments in both --pytest-arg forms" +trace MSYS_NO_PATHCONV=1 $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg -m --pytest-arg smoke,fast +trace print_requests.py //updates + +title "Relative paths resolve from the working directory" +cd tests +trace $CLI pipelines test my_pipeline test_orders.py::test_valid_order +cd .. +trace print_requests.py //updates + +title "Test selections require the pipeline KEY first" +trace musterr $CLI pipelines test tests/test_orders.py diff --git a/acceptance/pipelines/test/selection/tests/test_orders.py b/acceptance/pipelines/test/selection/tests/test_orders.py new file mode 100644 index 00000000000..6058cdd0bba --- /dev/null +++ b/acceptance/pipelines/test/selection/tests/test_orders.py @@ -0,0 +1,6 @@ +def test_valid_order(): + assert True + + +def test_backfill(): + assert True diff --git a/acceptance/pipelines/test/submit-error/out.test.toml b/acceptance/pipelines/test/submit-error/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/submit-error/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/submit-error/output.txt b/acceptance/pipelines/test/submit-error/output.txt new file mode 100644 index 00000000000..fc3de068652 --- /dev/null +++ b/acceptance/pipelines/test/submit-error/output.txt @@ -0,0 +1,13 @@ + +=== A submit error from the service is shown as is, and no update ID is printed +>>> musterr [CLI] pipelines test --pipeline-id [UUID][0] +Error: An active update '[UUID][1]' already exists for pipeline '[UUID][0]'. + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID][0]/updates", + "body": { + "test_only": true + } +} diff --git a/acceptance/pipelines/test/submit-error/script b/acceptance/pipelines/test/submit-error/script new file mode 100644 index 00000000000..0d0843028c6 --- /dev/null +++ b/acceptance/pipelines/test/submit-error/script @@ -0,0 +1,3 @@ +title "A submit error from the service is shown as is, and no update ID is printed" +trace musterr $CLI pipelines test --pipeline-id 11111111-2222-3333-4444-555555555555 +trace print_requests.py //updates diff --git a/acceptance/pipelines/test/submit-error/test.toml b/acceptance/pipelines/test/submit-error/test.toml new file mode 100644 index 00000000000..8932d1cfc26 --- /dev/null +++ b/acceptance/pipelines/test/submit-error/test.toml @@ -0,0 +1,9 @@ +[[Server]] +Pattern = "POST /api/2.0/pipelines/{pipeline_id}/updates" +Response.Body = ''' +{ + "error_code": "RESOURCE_CONFLICT", + "message": "An active update '00000000-0000-0000-0000-000000000001' already exists for pipeline '11111111-2222-3333-4444-555555555555'." +} +''' +Response.StatusCode = 409 diff --git a/acceptance/pipelines/test/test.toml b/acceptance/pipelines/test/test.toml new file mode 100644 index 00000000000..66f23523071 --- /dev/null +++ b/acceptance/pipelines/test/test.toml @@ -0,0 +1,4 @@ +RecordRequests = true +Ignore = ['.databricks'] +# The command is experimental; the gate case in arguments turns it off. +Env.DATABRICKS_CLI_PIPELINES_TEST = "true" diff --git a/cmd/pipelines/commands.go b/cmd/pipelines/commands.go index 6a6b6931313..24882f97864 100644 --- a/cmd/pipelines/commands.go +++ b/cmd/pipelines/commands.go @@ -17,6 +17,7 @@ func Commands() []*cobra.Command { destroyCommand(), runCommand(), dryRunCommand(), + testCommand(), historyCommand(), logsCommand(), openCommand(), diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go new file mode 100644 index 00000000000..dbbc9e042e8 --- /dev/null +++ b/cmd/pipelines/test.go @@ -0,0 +1,416 @@ +package pipelines + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "maps" + "net/http" + "os" + "path" + "path/filepath" + "slices" + "strings" + "uuid" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/resources" + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdgroup" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/shellquote" + databricks "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// startTestUpdateRequest is the Start Update body of a test run. +// test_only and test_options are not in the Go SDK yet, so the request is sent directly. +type startTestUpdateRequest struct { + TestOnly bool `json:"test_only"` + TestOptions *testOptions `json:"test_options,omitempty"` +} + +type testOptions struct { + TestSelection []string `json:"test_selection,omitempty"` + PytestArgs []string `json:"pytest_args,omitempty"` +} + +func newStartTestUpdateRequest(selection, pytestArgs []string) startTestUpdateRequest { + req := startTestUpdateRequest{TestOnly: true} + if len(selection) > 0 || len(pytestArgs) > 0 { + req.TestOptions = &testOptions{TestSelection: selection, PytestArgs: pytestArgs} + } + return req +} + +// pipelinesTestEnvVar enables the experimental pipelines test command. +const pipelinesTestEnvVar = "DATABRICKS_CLI_PIPELINES_TEST" + +const pytestArgHint = "pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow'" + +// validatePytestArgs accepts only -k and -m, each followed by its value in the next argument, as the API +// requires. The value cannot be empty or start with "-", because pytest reads it as an option. +func validatePytestArgs(args []string) error { + for i := 0; i < len(args); i++ { + arg := args[i] + switch { + case strings.TrimSpace(arg) == "": + return errors.New("--pytest-arg requires a non-empty value") + case arg == "-k" || arg == "-m": + if i == len(args)-1 { + return fmt.Errorf("pytest argument %s requires a value: pass it in the next --pytest-arg", arg) + } + next := args[i+1] + if strings.TrimSpace(next) == "" { + return fmt.Errorf("pytest argument %s requires a non-empty value", arg) + } + if strings.HasPrefix(next, "-") { + return fmt.Errorf("pytest argument %s requires a value, not %q: pass it in the next --pytest-arg", arg, next) + } + i++ + case strings.HasPrefix(arg, "-k") || strings.HasPrefix(arg, "-m"): + option, value := arg[:2], strings.TrimPrefix(arg[2:], "=") + return fmt.Errorf("pytest argument %q must be split: pass --pytest-arg=%s --pytest-arg=%s", arg, option, shellquote.BashArg(value)) + case !strings.HasPrefix(arg, "-"): + return fmt.Errorf("unsupported pytest argument %q: pass test paths and test IDs as TEST_SELECTION arguments", arg) + default: + return fmt.Errorf("unsupported pytest argument %q: only -k and -m are supported", arg) + } + } + return nil +} + +// testFlagErrorFunc adds guidance when pytest's -k or -m is passed as a CLI flag. +func testFlagErrorFunc(cmd *cobra.Command, err error) error { + if notExist, ok := errors.AsType[*pflag.NotExistError](err); ok { + if name := notExist.GetSpecifiedName(); name == "k" || name == "m" { + return fmt.Errorf("-%s is a pytest argument: %s", name, pytestArgHint) + } + } + return cmd.Parent().FlagErrorFunc()(cmd, err) +} + +// resolveTestSelections converts relative selection paths with toWorkspacePath and keeps +// the pytest test ID after "::" unchanged. Paths that start with "/" are workspace paths and are +// passed as typed: the service checks them and reports errors about what the user wrote. +func resolveTestSelections(selections []string, toWorkspacePath func(string) (string, error)) ([]string, error) { + var resolved []string + for _, selection := range selections { + p, testID, _ := strings.Cut(selection, "::") + if p == "" { + return nil, fmt.Errorf("test selection %q has no path", selection) + } + if !strings.HasPrefix(p, "/") { + var err error + p, err = toWorkspacePath(p) + if err != nil { + return nil, fmt.Errorf("cannot resolve test selection %q: %w", selection, err) + } + } + if strings.Contains(selection, "::") { + p += "::" + testID + } + resolved = append(resolved, p) + } + return resolved, nil +} + +// bundleWorkspacePath maps a local path, relative to the working directory, to its deployed +// path under filesRoot, the deployed path of the bundle's synchronized files. +func bundleWorkspacePath(cwd, syncRoot, filesRoot, localPath string) (string, error) { + if !filepath.IsAbs(localPath) { + localPath = filepath.Join(cwd, localPath) + } + rel, err := filepath.Rel(syncRoot, localPath) + if err != nil || !filepath.IsLocal(rel) { + return "", fmt.Errorf("%s is outside the synchronized bundle files in %s", filepath.ToSlash(localPath), filepath.ToSlash(syncRoot)) + } + return path.Join(filesRoot, filepath.ToSlash(rel)), nil +} + +// bundleFilesRoot returns the deployed path of the bundle's synchronized files, as TranslatePaths +// maps resource paths, or "" for an immutable deployment, whose snapshot path the command does not know. +func bundleFilesRoot(b *bundle.Bundle) string { + switch { + case b.SyncMode() == config.SyncModeImmutable: + return "" + case b.IsSourceLinkedDeployment(): + return b.SyncRootPath + default: + return b.Config.Workspace.FilePath + } +} + +// bundleTestSelectionPath maps a local path, relative to the working directory, to its deployed path +// under filesRoot. The path must exist locally, so that a typo fails before compute starts. +func bundleTestSelectionPath(cwd, syncRoot, filesRoot, localPath string) (string, error) { + if filesRoot == "" { + return "", errors.New("local test paths are not supported with workspace.sync_mode immutable; pass an absolute workspace path") + } + workspacePath, err := bundleWorkspacePath(cwd, syncRoot, filesRoot, localPath) + if err != nil { + return "", err + } + if !filepath.IsAbs(localPath) { + localPath = filepath.Join(cwd, localPath) + } + if _, err := os.Stat(localPath); errors.Is(err, fs.ErrNotExist) { + return "", fmt.Errorf("%s does not exist; pass a local path or an absolute workspace path", filepath.ToSlash(localPath)) + } else if err != nil { + return "", err + } + return workspacePath, nil +} + +// hasRelativeSelection reports whether a selection is a relative path. +func hasRelativeSelection(selections []string) bool { + for _, selection := range selections { + if p, _, _ := strings.Cut(selection, "::"); !strings.HasPrefix(p, "/") { + return true + } + } + return false +} + +// rootPathSelection appends a relative path to the pipeline's root_path without cleaning it, +// so that the service checks the path as the user typed it. +func rootPathSelection(rootPath, relPath string) (string, error) { + if rootPath == "" { + return "", errors.New("the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") + } + return strings.TrimSuffix(rootPath, "/") + "/" + filepath.ToSlash(relPath), nil +} + +// getRootPath reads the root_path of the deployed pipeline. +func getRootPath(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string) (string, error) { + pipeline, err := w.Pipelines.Get(ctx, pipelines.GetPipelineRequest{PipelineId: pipelineID}) + if err != nil { + return "", err + } + if pipeline.Spec == nil { + return "", nil + } + return pipeline.Spec.RootPath, nil +} + +// startTestUpdate starts a test-only update and returns its ID. +func startTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string, req startTestUpdateRequest) (string, error) { + apiClient, err := client.New(w.Config) + if err != nil { + return "", fmt.Errorf("failed to create API client: %w", err) + } + + var response pipelines.StartUpdateResponse + err = apiClient.Do( + ctx, + http.MethodPost, + fmt.Sprintf("/api/2.0/pipelines/%s/updates", pipelineID), + auth.WorkspaceIDHeaders(w.Config), + nil, + req, + &response, + ) + if err != nil { + return "", err + } + return response.UpdateId, nil +} + +// testTarget is the pipeline a test run is submitted to. +type testTarget struct { + key string + pipelineID string + selections []string + w *databricks.WorkspaceClient +} + +func (t testTarget) printHeader(out io.Writer, updateID string) { + if t.key != "" { + fmt.Fprintf(out, "Pipeline: %s (%s)\n", t.key, t.pipelineID) + } else { + fmt.Fprintf(out, "Pipeline: %s\n", t.pipelineID) + } + fmt.Fprintf(out, "Update: %s\n", updateID) +} + +// resolveTestTargetByID identifies the pipeline by --pipeline-id; all positional arguments are selections. +func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) (testTarget, error) { + if cmd.Flag("target").Value.String() == "" { + cmd.SetContext(root.SkipLoadBundle(cmd.Context())) + } + if err := root.MustWorkspaceClient(cmd, args); err != nil { + return testTarget{}, err + } + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + var rootPath string + if hasRelativeSelection(args) { + var err error + rootPath, err = getRootPath(ctx, w, pipelineID) + if err != nil { + return testTarget{}, err + } + } + selections, err := resolveTestSelections(args, func(p string) (string, error) { + return rootPathSelection(rootPath, p) + }) + if err != nil { + return testTarget{}, err + } + return testTarget{pipelineID: pipelineID, selections: selections, w: w}, nil +} + +// resolveTestTargetInBundle identifies the pipeline by its bundle KEY, which must precede any selections. +func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, error) { + var key string + b, err := utils.ProcessBundle(cmd, utils.ProcessOptions{ + PostInitFunc: func(ctx context.Context, b *bundle.Bundle) error { + var err error + key, err = resolvePipelineArgument(ctx, b, args[:min(len(args), 1)]) + return err + }, + ErrorOnEmptyState: true, + SkipInitContext: true, + }) + if err != nil { + return testTarget{}, err + } + ctx := cmd.Context() + + pipelineID, err := resolvePipelineIdFromKey(ctx, b, key) + if err != nil { + if _, lookupErr := resources.Lookup(b, key, isPipeline); lookupErr != nil { + return testTarget{}, notPipelineKeyError(cmd, b, key, args) + } + return testTarget{}, err + } + suggestPipelineDeploy(ctx, cmd) + + cwd, err := os.Getwd() + if err != nil { + return testTarget{}, err + } + var selectionArgs []string + if len(args) > 1 { + selectionArgs = args[1:] + } + filesRoot := bundleFilesRoot(b) + selections, err := resolveTestSelections(selectionArgs, func(p string) (string, error) { + return bundleTestSelectionPath(cwd, b.SyncRootPath, filesRoot, p) + }) + if err != nil { + return testTarget{}, err + } + return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx)}, nil +} + +// notPipelineKeyError explains that the first argument is not a pipeline KEY, names the bundle's +// pipelines, and shows the command that passes the arguments as test selections of the first one. +func notPipelineKeyError(cmd *cobra.Command, b *bundle.Bundle, key string, args []string) error { + keys := slices.Sorted(maps.Keys(resources.Completions(b, isPipeline))) + if len(keys) == 0 { + return fmt.Errorf("%q is not a pipeline in this bundle, and the bundle has no pipelines", key) + } + usage := []string{cmd.CommandPath(), shellquote.BashArg(keys[0])} + for _, arg := range args { + usage = append(usage, shellquote.BashArg(arg)) + } + return fmt.Errorf("%q is not a pipeline in this bundle (pipelines: %s). Usage: %s", key, strings.Join(keys, ", "), strings.Join(usage, " ")) +} + +func testCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "test [flags] [KEY] [TEST_SELECTION...]", + Short: "*Preview* Run the tests of a pipeline", + Long: `Run the tests of a pipeline in a test-only update. + +This command is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it. + +KEY is the unique name of the pipeline, as defined in its YAML file. +It is required when test selections are given. If there is only one pipeline +in the project and no arguments are given, it is selected automatically. +Outside a project, identify the pipeline with --pipeline-id; all positional +arguments are then test selections. + +TEST_SELECTION is a test directory, file, or pytest test ID, for example +tests/test_orders.py::test_backfill. In a project, a relative path is a local +path, relative to the working directory, and is converted to its deployed +workspace path. With --pipeline-id, a relative path is relative to the +pipeline's root_path. Paths that start with / are workspace paths. Without a +selection, all of the pipeline's tests run. + +Each --pytest-arg passes one argument to pytest, in the order given. Only -k +and -m are supported, each followed by its value in the next --pytest-arg, for +example: + + databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow'`, + } + + var pipelineID string + var pytestArgs []string + + testGroup := cmdgroup.NewFlagGroup("Pipeline Test") + testGroup.FlagSet().StringVar(&pipelineID, "pipeline-id", "", "ID of the pipeline to test, instead of a KEY.") + testGroup.FlagSet().StringArrayVar(&pytestArgs, "pytest-arg", nil, "Argument to pass to pytest (repeatable). Only -k and -m are supported.") + + wrappedCmd := cmdgroup.NewCommandWithGroupFlag(cmd) + wrappedCmd.AddFlagGroup(testGroup) + + cmd.SetFlagErrorFunc(testFlagErrorFunc) + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + if enabled, _ := env.GetBool(cmd.Context(), pipelinesTestEnvVar); !enabled { + return fmt.Errorf("pipelines test is experimental. Set %s=true to enable it", pipelinesTestEnvVar) + } + if cmd.ArgsLenAtDash() >= 0 { + return errors.New("the -- separator is not supported: " + pytestArgHint) + } + if root.OutputType(cmd) != flags.OutputText { + return fmt.Errorf("--output %s is not supported by pipelines test", root.OutputType(cmd)) + } + if err := validatePytestArgs(pytestArgs); err != nil { + return err + } + var target testTarget + if pipelineID != "" { + id, err := uuid.Parse(pipelineID) + if err != nil { + return fmt.Errorf("--pipeline-id %q is not a pipeline ID", pipelineID) + } + target, err = resolveTestTargetByID(cmd, args, id.String()) + if err != nil { + return err + } + } else { + cmd.SetContext(logdiag.InitContext(cmd.Context())) + var err error + target, err = resolveTestTargetInBundle(cmd, args) + if err != nil { + return err + } + } + ctx := cmd.Context() + + req := newStartTestUpdateRequest(target.selections, pytestArgs) + updateID, err := startTestUpdate(ctx, target.w, target.pipelineID, req) + if err != nil { + return err + } + target.printHeader(cmd.OutOrStdout(), updateID) + return nil + } + + return cmd +} diff --git a/cmd/pipelines/test_events.go b/cmd/pipelines/test_events.go new file mode 100644 index 00000000000..e9dd88aff70 --- /dev/null +++ b/cmd/pipelines/test_events.go @@ -0,0 +1,317 @@ +package pipelines + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + "github.com/databricks/cli/libs/auth" + databricks "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/pipelines" +) + +const ( + testCaseProgressEventType = "test_case_progress" + testRunSummaryEventType = "test_run_summary" +) + +type testCaseStatus string + +const ( + testCasePending testCaseStatus = "PENDING" + testCaseRunning testCaseStatus = "RUNNING" + testCasePassed testCaseStatus = "PASSED" + testCaseFailed testCaseStatus = "FAILED" + testCaseSkipped testCaseStatus = "SKIPPED" + testCaseError testCaseStatus = "ERROR" +) + +// parseTestCaseStatus accepts both the short status and the proto enum value name. +func parseTestCaseStatus(s string) testCaseStatus { + return testCaseStatus(strings.TrimPrefix(s, "PIPELINE_TEST_CASE_STATUS_")) +} + +func (s testCaseStatus) isFinal() bool { + switch s { + case testCasePassed, testCaseFailed, testCaseSkipped, testCaseError: + return true + default: + return false + } +} + +type testRunSummary struct { + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Skipped int `json:"skipped"` + Errored int `json:"errored"` +} + +// testPipelineEvent is an entry of List Pipeline Events. The Go SDK PipelineEvent has no +// details or origin.test_case_id, so test events are decoded from the raw response. +type testPipelineEvent struct { + Id string `json:"id"` + EventType string `json:"event_type"` + Timestamp string `json:"timestamp"` + Origin struct { + TestCaseId string `json:"test_case_id"` + } `json:"origin"` + Error *pipelines.ErrorDetail `json:"error"` + Truncation *pipelines.Truncation `json:"truncation"` + Details struct { + TestCaseProgress *struct { + Status string `json:"status"` + } `json:"test_case_progress"` + TestRunSummary *testRunSummary `json:"test_run_summary"` + } `json:"details"` +} + +func (e testPipelineEvent) isTruncated(field string) bool { + if e.Truncation == nil { + return false + } + for _, f := range e.Truncation.TruncatedFields { + if f.FieldName == field { + return true + } + } + return false +} + +// testEventsPageSize is the largest page List Pipeline Events returns. +const testEventsPageSize = 250 + +// readTestEvents returns the events of one update in timestamp order, following every page. With +// sinceTimestamp it returns only the events at or after that timestamp, which must be a timestamp +// string exactly as the API returned it. +func readTestEvents(ctx context.Context, w *databricks.WorkspaceClient, pipelineID, updateID, sinceTimestamp string) ([]testPipelineEvent, error) { + apiClient, err := client.New(w.Config) + if err != nil { + return nil, fmt.Errorf("failed to create API client: %w", err) + } + path := fmt.Sprintf("/api/2.0/pipelines/%s/events", pipelineID) + request := pipelines.ListPipelineEventsRequest{ + Filter: buildPipelineEventFilter(updateID, nil, nil, sinceTimestamp, ""), + OrderBy: []string{"timestamp asc"}, + MaxResults: testEventsPageSize, + } + var events []testPipelineEvent + for { + var page struct { + Events []testPipelineEvent `json:"events"` + NextPageToken string `json:"next_page_token"` + } + err := apiClient.Do(ctx, http.MethodGet, path, auth.WorkspaceIDHeaders(w.Config), nil, request, &page) + if err != nil { + return nil, fmt.Errorf("failed to fetch pipeline events: %w", err) + } + events = append(events, page.Events...) + if page.NextPageToken == "" || len(page.Events) == 0 { + return events, nil + } + // A request with page_token must not repeat filter or order_by: the token carries them, + // and the service rejects the request otherwise. + request = pipelines.ListPipelineEventsRequest{PageToken: page.NextPageToken, MaxResults: testEventsPageSize} + } +} + +// testCaseResult is what the events report about one test case. +type testCaseResult struct { + // id is origin.test_case_id: the absolute workspace path of the test file followed by the pytest + // test ID, such as /Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill. + id string + status testCaseStatus + // finals counts final outcomes; a complete result has exactly one per case. + finals int + started time.Time + finished time.Time + errorDetail *pipelines.ErrorDetail + detailsTruncated bool +} + +// duration returns the time between the RUNNING and the final event, if both are known. +func (c *testCaseResult) duration() (time.Duration, bool) { + if c.started.IsZero() || c.finished.IsZero() || c.finished.Before(c.started) { + return 0, false + } + return c.finished.Sub(c.started), true +} + +// testRun accumulates the events of a test update, ignoring events it has already seen. +type testRun struct { + seen map[string]bool + cases []*testCaseResult + byID map[string]*testCaseResult + summaries []testRunSummary + // unreadable counts test events whose outcome is missing. + unreadable int + // fatalError is the first fatal error that the update reported, other than the test verdict. + fatalError string + // verdictError is the service's verdict that tests failed, if the update reported it. + verdictError string +} + +// testVerdictErrorPrefix starts the message of the error by which the service reports that tests +// failed, such as "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed (1 passed, 0 skipped).". +// The message is matched because ErrorDetail has no error class yet; only the output depends on it. +const testVerdictErrorPrefix = "[PIPELINE_TEST_FAILED]" + +// isTestVerdict reports whether an update error is the service's verdict that tests failed. +func isTestVerdict(e *pipelines.ErrorDetail) bool { + for _, exception := range e.Exceptions { + if strings.HasPrefix(exception.Message, testVerdictErrorPrefix) { + return true + } + } + return false +} + +func newTestRun() *testRun { + return &testRun{seen: map[string]bool{}, byID: map[string]*testCaseResult{}} +} + +// add records an event and returns the case it gave a final outcome to, if any. +func (r *testRun) add(e testPipelineEvent) *testCaseResult { + if r.seen[e.Id] { + return nil + } + r.seen[e.Id] = true + + if e.Error != nil && e.Error.Fatal { + switch { + case isTestVerdict(e.Error): + r.verdictError = errorDetailText(e.Error) + case r.fatalError == "": + r.fatalError = errorDetailText(e.Error) + } + } + + switch e.EventType { + case testRunSummaryEventType: + if e.Details.TestRunSummary == nil { + r.unreadable++ + return nil + } + r.summaries = append(r.summaries, *e.Details.TestRunSummary) + case testCaseProgressEventType: + progress := e.Details.TestCaseProgress + if progress == nil || e.Origin.TestCaseId == "" { + r.unreadable++ + return nil + } + status := parseTestCaseStatus(progress.Status) + if !status.isFinal() && status != testCasePending && status != testCaseRunning { + r.unreadable++ + return nil + } + + c, ok := r.byID[e.Origin.TestCaseId] + if !ok { + c = &testCaseResult{id: e.Origin.TestCaseId, status: testCasePending} + r.byID[c.id] = c + r.cases = append(r.cases, c) + } + timestamp, _ := time.Parse(time.RFC3339Nano, e.Timestamp) + switch { + case status == testCaseRunning: + c.started = timestamp + if !c.status.isFinal() { + c.status = testCaseRunning + } + case status.isFinal(): + c.finals++ + c.status = status + c.finished = timestamp + c.errorDetail = e.Error + c.detailsTruncated = e.isTruncated("error") + if c.finals == 1 { + return c + } + } + } + return nil +} + +// counts returns the number of cases with each final outcome. +func (r *testRun) counts() testRunSummary { + var s testRunSummary + for _, c := range r.cases { + if !c.status.isFinal() { + continue + } + s.Total++ + switch c.status { + case testCasePassed: + s.Passed++ + case testCaseFailed: + s.Failed++ + case testCaseSkipped: + s.Skipped++ + case testCaseError: + s.Errored++ + default: + } + } + return s +} + +// incompleteReason explains why the events of an ended update are not a complete result, +// or returns "" if they are: one summary and one final outcome per case, matching its counts. +func (r *testRun) incompleteReason(state pipelines.UpdateInfoState) string { + if r.unreadable > 0 { + return "some test results could not be read" + } + switch len(r.summaries) { + case 0: + switch state { + case pipelines.UpdateInfoStateFailed: + return "the update failed before all test results were received" + case pipelines.UpdateInfoStateCanceled: + return "the update was canceled before all test results were received" + default: + return "the update ended without a test summary" + } + case 1: + default: + return fmt.Sprintf("received %d test summaries instead of one", len(r.summaries)) + } + + for _, c := range r.cases { + if c.finals > 1 { + return fmt.Sprintf("test %s has more than one result", c.id) + } + } + summary := r.summaries[0] + received := r.counts() + if received.Total != summary.Total { + return fmt.Sprintf("received %d of %d test results", received.Total, summary.Total) + } + if received != summary { + return "the test results do not match the test summary" + } + for _, c := range r.cases { + if c.finals == 0 { + return fmt.Sprintf("test %s has no result", c.id) + } + } + return "" +} + +func errorDetailText(detail *pipelines.ErrorDetail) string { + var parts []string + for _, e := range detail.Exceptions { + switch { + case e.ClassName != "" && e.Message != "": + parts = append(parts, e.ClassName+": "+e.Message) + case e.Message != "": + parts = append(parts, e.Message) + case e.ClassName != "": + parts = append(parts, e.ClassName) + } + } + return strings.Join(parts, "\n") +} diff --git a/cmd/pipelines/test_events_test.go b/cmd/pipelines/test_events_test.go new file mode 100644 index 00000000000..fb0288b26c8 --- /dev/null +++ b/cmd/pipelines/test_events_test.go @@ -0,0 +1,254 @@ +package pipelines + +import ( + "encoding/json" + "fmt" + "net/url" + "strconv" + "strings" + "testing" + "time" + + "github.com/databricks/cli/libs/testserver" + databricks "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testEvents builds the events of one test, each with its own event ID. +type testEvents struct { + t *testing.T + n int +} + +func newTestEvents(t *testing.T) *testEvents { + return &testEvents{t: t} +} + +func (b *testEvents) testCase(id, status, timestamp string, extra ...string) testPipelineEvent { + b.n++ + return decodeTestEvent(b.t, fmt.Sprintf(`{ + "id": "e%d", "event_type": "test_case_progress", "timestamp": %q, + "origin": {"test_case_id": %q}, + "details": {"test_case_progress": {"status": %q}}%s + }`, b.n, timestamp, id, status, strings.Join(append([]string{""}, extra...), ","))) +} + +func (b *testEvents) summary(total, passed, failed, skipped, errored int) testPipelineEvent { + b.n++ + return decodeTestEvent(b.t, fmt.Sprintf(`{ + "id": "e%d", "event_type": "test_run_summary", + "details": {"test_run_summary": {"total": %d, "passed": %d, "failed": %d, "skipped": %d, "errored": %d}} + }`, b.n, total, passed, failed, skipped, errored)) +} + +func decodeTestEvent(t *testing.T, data string) testPipelineEvent { + var e testPipelineEvent + require.NoError(t, json.Unmarshal([]byte(data), &e)) + return e +} + +func TestTestRunAdd(t *testing.T) { + ev := newTestEvents(t) + run := newTestRun() + events := []testPipelineEvent{ + ev.testCase("tests/test_a.py::test_ok", "PENDING", "2026-01-01T00:00:00.000Z"), + ev.testCase("tests/test_a.py::test_ok", "RUNNING", "2026-01-01T00:00:01.000Z"), + ev.testCase("tests/test_a.py::test_ok", "PASSED", "2026-01-01T00:00:02.840Z"), + ev.testCase("tests/test_a.py::test_bad", "PIPELINE_TEST_CASE_STATUS_FAILED", "2026-01-01T00:00:03.000Z", + `"error": {"exceptions": [{"class_name": "AssertionError", "message": "assert 1 == 2"}]}`, + `"truncation": {"truncated_fields": [{"field_name": "error"}]}`), + // A late RUNNING event does not undo the final outcome. + ev.testCase("tests/test_a.py::test_bad", "RUNNING", "2026-01-01T00:00:02.000Z"), + ev.summary(2, 1, 1, 0, 0), + } + + var finished []string + for _, e := range events { + if c := run.add(e); c != nil { + finished = append(finished, c.id) + } + } + // Reading every page again adds nothing. + for _, e := range events { + assert.Nil(t, run.add(e)) + } + + assert.Equal(t, []string{"tests/test_a.py::test_ok", "tests/test_a.py::test_bad"}, finished) + require.Len(t, run.cases, 2) + + ok := run.cases[0] + assert.Equal(t, testCasePassed, ok.status) + d, known := ok.duration() + assert.True(t, known) + assert.Equal(t, 1840*time.Millisecond, d) + + bad := run.cases[1] + assert.Equal(t, testCaseFailed, bad.status) + assert.True(t, bad.detailsTruncated) + assert.Equal(t, "AssertionError: assert 1 == 2", errorDetailText(bad.errorDetail)) + d, known = bad.duration() + assert.True(t, known) + assert.Equal(t, time.Second, d) + + assert.Empty(t, run.incompleteReason(pipelines.UpdateInfoStateFailed)) +} + +func TestTestRunIncompleteReason(t *testing.T) { + ev := newTestEvents(t) + passed := func() testPipelineEvent { + return ev.testCase("t::a", "PASSED", "") + } + tests := []struct { + name string + events []testPipelineEvent + state pipelines.UpdateInfoState + want string + }{ + { + name: "no summary after completion", + events: []testPipelineEvent{passed()}, + state: pipelines.UpdateInfoStateCompleted, + want: "the update ended without a test summary", + }, + { + name: "two summaries", + events: []testPipelineEvent{passed(), ev.summary(1, 1, 0, 0, 0), ev.summary(1, 1, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "received 2 test summaries instead of one", + }, + { + name: "missing outcome", + events: []testPipelineEvent{passed(), ev.testCase("t::b", "RUNNING", ""), ev.summary(2, 2, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "received 1 of 2 test results", + }, + { + name: "case without outcome that the summary leaves out", + events: []testPipelineEvent{passed(), ev.testCase("t::b", "RUNNING", ""), ev.summary(1, 1, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "test t::b has no result", + }, + { + name: "two outcomes for one case", + events: []testPipelineEvent{passed(), ev.testCase("t::a", "FAILED", ""), ev.summary(1, 1, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "test t::a has more than one result", + }, + { + name: "outcomes differ from summary", + events: []testPipelineEvent{ev.testCase("t::a", "FAILED", ""), ev.summary(1, 1, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "the test results do not match the test summary", + }, + { + name: "outcome without details", + events: []testPipelineEvent{ + passed(), + decodeTestEvent(t, `{"id": "truncated", "event_type": "test_case_progress", "origin": {"test_case_id": "t::b"}}`), + ev.summary(1, 1, 0, 0, 0), + }, + state: pipelines.UpdateInfoStateCompleted, + want: "some test results could not be read", + }, + { + name: "unknown status", + events: []testPipelineEvent{passed(), ev.testCase("t::b", "PIPELINE_TEST_CASE_STATUS_UNSPECIFIED", ""), ev.summary(1, 1, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "some test results could not be read", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + run := newTestRun() + for _, e := range tc.events { + run.add(e) + } + assert.Equal(t, tc.want, run.incompleteReason(tc.state)) + }) + } +} + +func TestReadTestEventsFollowsPages(t *testing.T) { + server := testserver.New(t) + var queries []url.Values + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}/events", func(req testserver.Request) any { + queries = append(queries, req.URL.Query()) + if req.URL.Query().Get("page_token") == "" { + return map[string]any{"events": []map[string]string{{"id": "e1"}}, "next_page_token": "page2"} + } + return map[string]any{"events": []map[string]string{{"id": "e2"}}} + }) + + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + events, err := readTestEvents(t.Context(), w, "p1", "u1", "2026-01-01T00:00:01.000Z") + require.NoError(t, err) + + var ids []string + for _, e := range events { + ids = append(ids, e.Id) + } + assert.Equal(t, []string{"e1", "e2"}, ids) + assert.Equal(t, []url.Values{ + { + "filter": {"update_id = 'u1' AND timestamp >= '2026-01-01T00:00:01.000Z'"}, + "order_by": {"timestamp asc"}, + "max_results": {"250"}, + }, + { + "page_token": {"page2"}, + "max_results": {"250"}, + }, + }, queries) +} + +// NFR3: a run of 200 cases spans several pages, and each case is counted once. +func TestReadTestEvents200Cases(t *testing.T) { + const cases = 200 + var all []map[string]any + for i := range cases { + id := fmt.Sprintf("/Workspace/p/tests/test_many.py::test_case_%03d", i) + for j, status := range []string{"PENDING", "RUNNING", "PASSED"} { + all = append(all, map[string]any{ + "id": fmt.Sprintf("c%d-%d", i, j), "event_type": "test_case_progress", "timestamp": "2026-01-01T00:00:00.000Z", + "origin": map[string]any{"test_case_id": id}, "details": map[string]any{"test_case_progress": map[string]any{"status": status}}, + }) + } + } + all = append(all, map[string]any{ + "id": "summary", "event_type": "test_run_summary", + "details": map[string]any{"test_run_summary": map[string]any{"total": cases, "passed": cases}}, + }) + + server := testserver.New(t) + pages := 0 + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}/events", func(req testserver.Request) any { + pages++ + offset := 0 + if token := req.URL.Query().Get("page_token"); token != "" { + _, err := fmt.Sscan(token, &offset) + require.NoError(t, err) + } + end := min(offset+testEventsPageSize, len(all)) + page := map[string]any{"events": all[offset:end]} + if end < len(all) { + page["next_page_token"] = strconv.Itoa(end) + } + return page + }) + + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + events, err := readTestEvents(t.Context(), w, "p1", "u1", "") + require.NoError(t, err) + assert.Equal(t, 3, pages) + + run := newTestRun() + for _, e := range append(events, events...) { + run.add(e) + } + assert.Empty(t, run.incompleteReason(pipelines.UpdateInfoStateCompleted)) + assert.Equal(t, testRunSummary{Total: cases, Passed: cases}, run.counts()) +} diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go new file mode 100644 index 00000000000..682567d5b06 --- /dev/null +++ b/cmd/pipelines/test_test.go @@ -0,0 +1,127 @@ +package pipelines + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/spf13/cobra" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidatePytestArgs(t *testing.T) { + tests := []struct { + args []string + wantErr string + }{ + {args: nil}, + {args: []string{""}, wantErr: "--pytest-arg requires a non-empty value"}, + {args: []string{"-k", "orders and not slow", "-m", "smoke,fast"}}, + {args: []string{"-korders"}, wantErr: `pytest argument "-korders" must be split: pass --pytest-arg=-k --pytest-arg=orders`}, + {args: []string{"-k=orders and not slow"}, wantErr: `pytest argument "-k=orders and not slow" must be split: pass --pytest-arg=-k --pytest-arg='orders and not slow'`}, + {args: []string{"-k", " "}, wantErr: "pytest argument -k requires a non-empty value"}, + {args: []string{"-k", "-m"}, wantErr: `pytest argument -k requires a value, not "-m": pass it in the next --pytest-arg`}, + {args: []string{"-k"}, wantErr: "pytest argument -k requires a value: pass it in the next --pytest-arg"}, + {args: []string{"-x"}, wantErr: `unsupported pytest argument "-x": only -k and -m are supported`}, + {args: []string{"--collect-only"}, wantErr: `unsupported pytest argument "--collect-only": only -k and -m are supported`}, + {args: []string{"tests/test_orders.py"}, wantErr: `unsupported pytest argument "tests/test_orders.py": pass test paths and test IDs as TEST_SELECTION arguments`}, + } + for _, tc := range tests { + err := validatePytestArgs(tc.args) + if tc.wantErr == "" { + assert.NoError(t, err, tc.args) + } else { + assert.EqualError(t, err, tc.wantErr, tc.args) + } + } +} + +func TestResolveTestSelections(t *testing.T) { + toWorkspacePath := func(p string) (string, error) { + if p == "missing" { + return "", errors.New("not found") + } + return "/Workspace/root/" + p, nil + } + + resolved, err := resolveTestSelections([]string{ + "tests", + "tests/test_orders.py::TestOrders::test_backfill[a::b]", + "/Workspace/Users/someone/../someone/tests/test_x.py::test_x", + }, toWorkspacePath) + require.NoError(t, err) + assert.Equal(t, []string{ + "/Workspace/root/tests", + "/Workspace/root/tests/test_orders.py::TestOrders::test_backfill[a::b]", + "/Workspace/Users/someone/../someone/tests/test_x.py::test_x", + }, resolved) + + _, err = resolveTestSelections([]string{"::test_x"}, toWorkspacePath) + assert.EqualError(t, err, `test selection "::test_x" has no path`) + + _, err = resolveTestSelections([]string{"missing::test_x"}, toWorkspacePath) + assert.EqualError(t, err, `cannot resolve test selection "missing::test_x": not found`) +} + +func TestNewStartTestUpdateRequest(t *testing.T) { + assert.Equal(t, startTestUpdateRequest{TestOnly: true}, newStartTestUpdateRequest(nil, nil)) + assert.Equal(t, startTestUpdateRequest{ + TestOnly: true, + TestOptions: &testOptions{TestSelection: []string{"/Workspace/tests"}, PytestArgs: []string{"-k", "orders"}}, + }, newStartTestUpdateRequest([]string{"/Workspace/tests"}, []string{"-k", "orders"})) +} + +func TestRootPathSelection(t *testing.T) { + got, err := rootPathSelection("/Workspace/Users/someone/orders/", "../other/tests") + require.NoError(t, err) + // The path is not cleaned, so the service rejects ".." with its own error. + assert.Equal(t, "/Workspace/Users/someone/orders/../other/tests", got) + + _, err = rootPathSelection("", "tests") + assert.ErrorContains(t, err, "the pipeline has no root_path") +} + +// Source-linked deployments run the source files, as TranslatePaths maps them. +func TestBundleFilesRootSourceLinked(t *testing.T) { + b := &bundle.Bundle{SyncRootPath: "/Workspace/Users/someone/orders"} + b.Config.Workspace.FilePath = "/Workspace/Users/someone/.bundle/orders/dev/files" + b.Config.Workspace.SyncMode = config.SyncModeInPlace + assert.Equal(t, "/Workspace/Users/someone/orders", bundleFilesRoot(b)) +} + +func TestBundleTestSelectionPath(t *testing.T) { + syncRoot := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(syncRoot, "pipeline", "tests"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(syncRoot, "conftest.py"), nil, 0o644)) + filesRoot := "/Workspace/Users/someone/.bundle/orders/dev/files" + + // A path may leave the working directory as long as it stays in the synchronized files. + got, err := bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline", "tests"), syncRoot, filesRoot, filepath.Join("..", "..", "conftest.py")) + require.NoError(t, err) + assert.Equal(t, filesRoot+"/conftest.py", got) + + // A path that does not exist locally is an error, so that a typo fails before compute starts. + _, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, "tests/test_a.py") + assert.EqualError(t, err, filepath.ToSlash(filepath.Join(syncRoot, "tests", "test_a.py"))+" does not exist; pass a local path or an absolute workspace path") + + // A path outside the synchronized files is an error. The error shows forward slashes. + _, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, filepath.Join("..", "tests")) + assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+filepath.ToSlash(syncRoot)) +} + +func TestNotPipelineKeyError(t *testing.T) { + cmd := &cobra.Command{Use: "test"} + b := &bundle.Bundle{} + b.Config.Resources.Pipelines = map[string]*resources.Pipeline{"orders": {}, "customers": {}} + err := notPipelineKeyError(cmd, b, "tests/test_orders.py", []string{"tests/test_orders.py", "tests/test_x.py::test_a b"}) + assert.EqualError(t, err, `"tests/test_orders.py" is not a pipeline in this bundle (pipelines: customers, orders). Usage: test customers tests/test_orders.py 'tests/test_x.py::test_a b'`) + + err = notPipelineKeyError(cmd, &bundle.Bundle{}, "tests", []string{"tests"}) + assert.EqualError(t, err, `"tests" is not a pipeline in this bundle, and the bundle has no pipelines`) +}