Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions acceptance/bundle/refschema/output/pipelines.txt
Original file line number Diff line number Diff line change
Expand Up @@ -867,6 +867,14 @@ resources.pipelines.*.storage string ALL
resources.pipelines.*.tags map[string]string ALL
resources.pipelines.*.tags.* string ALL
resources.pipelines.*.target string ALL
resources.pipelines.*.testing *resources.PipelineTesting INPUT
resources.pipelines.*.testing.environment *resources.PipelineTestEnvironment INPUT
resources.pipelines.*.testing.environment.dependencies []string INPUT
resources.pipelines.*.testing.environment.dependencies[*] string INPUT
resources.pipelines.*.testing.sources []resources.PipelineTestSource INPUT
resources.pipelines.*.testing.sources[*] resources.PipelineTestSource INPUT
resources.pipelines.*.testing.sources[*].glob *pipelines.PathPattern INPUT
resources.pipelines.*.testing.sources[*].glob.include string INPUT
resources.pipelines.*.trigger *pipelines.PipelineTrigger ALL
resources.pipelines.*.trigger.cron *pipelines.CronTrigger ALL
resources.pipelines.*.trigger.cron.quartz_cron_schedule string ALL
Expand Down
19 changes: 19 additions & 0 deletions acceptance/bundle/resources/pipelines/testing/databricks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
bundle:
name: pipeline-testing

resources:
pipelines:
orders:
name: orders
libraries:
- glob:
include: ./transformations/**
testing:
sources:
- glob:
include: ./tests/**
environment:
dependencies:
- -e .[test]
- -r ./tests/requirements.txt
- ./dist/*.whl
2 changes: 2 additions & 0 deletions acceptance/bundle/resources/pipelines/testing/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
33 changes: 33 additions & 0 deletions acceptance/bundle/resources/pipelines/testing/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@

>>> [CLI] bundle validate -o json
{
"environment": {
"dependencies": [
"-e .[test]",
"-r ./tests/requirements.txt",
"./dist/*.whl"
]
},
"sources": [
{
"glob": {
"include": "./tests/**"
}
}
]
}

=== Without pipeline testing, the property is an error

>>> musterr env -u DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING [CLI] bundle validate
Error: the testing property of pipelines is experimental. Set DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true to enable it
at resources.pipelines.orders.testing
in databricks.yml:12:9

Name: pipeline-testing
Target: default
Workspace:
User: [USERNAME]
Path: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default

Found 1 error
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
[project]
name = "orders"
version = "0.1"

[project.optional-dependencies]
test = ["faker"]
4 changes: 4 additions & 0 deletions acceptance/bundle/resources/pipelines/testing/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing

title "Without pipeline testing, the property is an error"
trace musterr env -u DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING $CLI bundle validate
5 changes: 5 additions & 0 deletions acceptance/bundle/resources/pipelines/testing/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Runs only against the local test server.
Cloud = false
RecordRequests = true
Ignore = [".databricks"]
Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true"
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
faker
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
def test_orders():
pass
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
from pyspark import pipelines as dp
28 changes: 28 additions & 0 deletions bundle/config/resources/pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,34 @@ type Pipeline struct {
// Nil (unset) uses the server default, which cascades. Set to false to retain datasets
// when the pipeline is deleted. Delete-time only: never sent on create/update.
CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"`

// Testing is PipelineSpec.testing, which the Go SDK does not have yet. Remove it, and the
// types below, once CreatePipeline has it.
Testing *PipelineTesting `json:"testing,omitempty"`
}

// PipelineTesting is the test configuration of a pipeline. Only test-only updates use it.
type PipelineTesting struct {
// Sources are the workspace files and folders that hold the pipeline's tests. They must
// not overlap the pipeline's libraries.
Sources []PipelineTestSource `json:"sources,omitempty"`

// Environment lists the packages that only the tests need.
Environment *PipelineTestEnvironment `json:"environment,omitempty"`
}

// PipelineTestSource is a workspace location that holds pipeline tests.
type PipelineTestSource struct {
// Glob is a test file, or a folder followed by /**.
Glob *pipelines.PathPattern `json:"glob,omitempty"`
}

// PipelineTestEnvironment lists the packages that a pipeline's tests need in addition to the
// pipeline's environment.
type PipelineTestEnvironment struct {
// Dependencies are files that declare the packages: a project folder, a requirements
// file, or a wheel. Package names and URLs are rejected by the API.
Dependencies []string `json:"dependencies,omitempty"`
}

func (p *Pipeline) UnmarshalJSON(b []byte) error {
Expand Down
49 changes: 49 additions & 0 deletions bundle/config/validate/pipeline_testing_enabled.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package validate

import (
"context"
"fmt"
"maps"
"slices"

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/env"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
)

type pipelineTestingEnabled struct{ bundle.RO }

// PipelineTestingEnabled rejects the testing property of pipelines while the experimental feature is off.
func PipelineTestingEnabled() bundle.ReadOnlyMutator {
return &pipelineTestingEnabled{}
}

func (v *pipelineTestingEnabled) Name() string {
return "validate:pipeline_testing_enabled"
}

func (v *pipelineTestingEnabled) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
if env.ExperimentalPipelineTesting(ctx) {
return nil
}
var diags diag.Diagnostics
pipelines := b.Config.Resources.Pipelines
for _, key := range slices.Sorted(maps.Keys(pipelines)) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

pipeline_development_deprecated.go right above this in initialize.go does the same walk with dyn.MapByPattern on resources.pipelines.*.development, which gives the path and locations directly. Can we do the same here for resources.pipelines.*.testing instead of iterating the typed map and calling dyn.GetByPath per key?

if pipelines[key] == nil || pipelines[key].Testing == nil {
continue
}
path := dyn.NewPath(dyn.Key("resources"), dyn.Key("pipelines"), dyn.Key(key), dyn.Key("testing"))
value, err := dyn.GetByPath(b.Config.Value(), path)
if err != nil {
return diag.FromErr(err)
}
diags = append(diags, diag.Diagnostic{
Severity: diag.Error,
Summary: fmt.Sprintf("the testing property of pipelines is experimental. Set %s=true to enable it", env.ExperimentalPipelineTestingVariable),
Locations: value.Locations(),
Paths: dyn.ToStructPaths(path),
})
}
return diags
}
41 changes: 41 additions & 0 deletions bundle/config/validate/pipeline_testing_enabled_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package validate

import (
"testing"

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/env"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/libs/diag"
libsenv "github.com/databricks/cli/libs/env"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestPipelineTestingEnabled(t *testing.T) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need this unit test? The acceptance test already shows the error with its location and path (the env -u ... bundle validate step) and the enabled case, and we try not to unit test mutators that an acceptance output covers.

b := &bundle.Bundle{
Config: config.Root{
Resources: config.Resources{
Pipelines: map[string]*resources.Pipeline{
"orders": {Testing: &resources.PipelineTesting{}},
"reports": {},
},
},
},
}
loc := diag.Location{File: "databricks.yml", Line: 9, Column: 9}
bundletest.SetLocation(b, "resources.pipelines.orders.testing", []diag.Location{loc})

diags := PipelineTestingEnabled().Apply(t.Context(), b)
require.Len(t, diags, 1)
assert.Equal(t, diag.Error, diags[0].Severity)
assert.Contains(t, diags[0].Summary, env.ExperimentalPipelineTestingVariable)
assert.Equal(t, []diag.Location{loc}, diags[0].Locations)
require.Len(t, diags[0].Paths, 1)
assert.Equal(t, "resources.pipelines.orders.testing", diags[0].Paths[0].String())

ctx := libsenv.Set(t.Context(), env.ExperimentalPipelineTestingVariable, "true")
assert.Empty(t, PipelineTestingEnabled().Apply(ctx, b))
}
4 changes: 4 additions & 0 deletions bundle/direct/dresources/type_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,10 @@ var knownMissingInStateType = map[string][]string{
"alerts": {
"file_path",
},
"pipelines": {
// The experimental testing property is not deployed yet.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this lands before #7438, a bundle with the variable on and testing set deploys fine and testing is silently dropped (PrepareState copies only CreatePipeline and CascadeOnDestroy). Fine for an experimental flag while the stack merges together, or do we want validate to say the property isn't deployed yet?

"testing",
},
"apps": {
"lifecycle.prevent_destroy",
"lifecycle.depends_on",
Expand Down
17 changes: 17 additions & 0 deletions bundle/env/experimental_pipeline_testing.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package env

import (
"context"

envlib "github.com/databricks/cli/libs/env"
)

// ExperimentalPipelineTestingVariable enables the experimental testing property of pipelines.
const ExperimentalPipelineTestingVariable = "DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING"

// ExperimentalPipelineTesting reports whether the experimental testing property of pipelines is
// enabled. While it is off, bundles deploy pipelines exactly as they did before the property existed.
func ExperimentalPipelineTesting(ctx context.Context) bool {
enabled, _ := envlib.GetBool(ctx, ExperimentalPipelineTestingVariable)
return enabled
}
20 changes: 20 additions & 0 deletions bundle/internal/schema/annotations.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1665,6 +1665,26 @@ resources:
"user_name":
"description": |-
The name of the user granted the permission level.
"testing":
"description": |-
The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. It is experimental and needs `DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true`.
"x-databricks-launch-stage": |-
PRIVATE_PREVIEW
"$fields":
"environment":
"description": |-
The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not.
"$fields":
"dependencies":
"description": |-
A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `-e` and an extra, for example `-e ./project[test]`. A relative path, such as `-e ./project[test]` or `-r ./tests/requirements.txt`, is relative to the YAML file that defines it. Package names and URLs are rejected.
"sources":
"description": |-
The workspace files and folders that hold the pipeline's tests.
"$fields":
"glob":
"description": |-
A `.py` test file, or a folder followed by `/**`. Other wildcards are rejected, and the sources must not overlap the pipeline's `libraries`. A relative path is relative to the YAML file that defines it.
"trigger":
"deprecation_message": |-
Use continuous instead
Expand Down
4 changes: 4 additions & 0 deletions bundle/phases/initialize.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,10 @@ func Initialize(ctx context.Context, b *bundle.Bundle) {
// Must run after PythonMutator to see development values set by Python mutators.
validate.PipelineDevelopmentDeprecated(),

// Reject the experimental testing property of pipelines while it is off. Must run after
// PythonMutator, which can set testing, and before TranslatePaths and the deploy use it.
validate.PipelineTestingEnabled(),

// Compute resources.volumes.*.volume_path and resolve references to it. Must run after
// PythonMutator: volume_path is computed and read-only, not declared by the PyDABs Volume
// model, so exposing it to Python would fail resource loading (like "deployment" below).
Expand Down
78 changes: 78 additions & 0 deletions bundle/schema/jsonschema.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading