Repository navigation
Add the testing property of pipelines to bundles #7300
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: tharun/pipelines-test-b1-editable-folder
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| bundle: | ||
| name: pipeline-testing | ||
|
|
||
| resources: | ||
| pipelines: | ||
| orders: | ||
| name: orders | ||
| libraries: | ||
| - glob: | ||
| include: ./transformations/** | ||
| testing: | ||
| sources: | ||
| - glob: | ||
| include: ./tests/** | ||
| environment: | ||
| dependencies: | ||
| - -e .[test] | ||
| - -r ./tests/requirements.txt | ||
| - ./dist/*.whl |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| Cloud = false | ||
| EnvMatrix.DMS = ["", "true"] |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
|
|
||
| >>> [CLI] bundle validate -o json | ||
| { | ||
| "environment": { | ||
| "dependencies": [ | ||
| "-e .[test]", | ||
| "-r ./tests/requirements.txt", | ||
| "./dist/*.whl" | ||
| ] | ||
| }, | ||
| "sources": [ | ||
| { | ||
| "glob": { | ||
| "include": "./tests/**" | ||
| } | ||
| } | ||
| ] | ||
| } | ||
|
|
||
| === Without pipeline testing, the property is an error | ||
|
|
||
| >>> musterr env -u DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING [CLI] bundle validate | ||
| Error: the testing property of pipelines is experimental. Set DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true to enable it | ||
| at resources.pipelines.orders.testing | ||
| in databricks.yml:12:9 | ||
|
|
||
| Name: pipeline-testing | ||
| Target: default | ||
| Workspace: | ||
| User: [USERNAME] | ||
| Path: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default | ||
|
|
||
| Found 1 error |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| [project] | ||
| name = "orders" | ||
| version = "0.1" | ||
|
|
||
| [project.optional-dependencies] | ||
| test = ["faker"] |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing | ||
|
|
||
| title "Without pipeline testing, the property is an error" | ||
| trace musterr env -u DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING $CLI bundle validate |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| # Runs only against the local test server. | ||
| Cloud = false | ||
| RecordRequests = true | ||
| Ignore = [".databricks"] | ||
| Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| faker |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| def test_orders(): | ||
| pass |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| from pyspark import pipelines as dp |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| package validate | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "maps" | ||
| "slices" | ||
|
|
||
| "github.com/databricks/cli/bundle" | ||
| "github.com/databricks/cli/bundle/env" | ||
| "github.com/databricks/cli/libs/diag" | ||
| "github.com/databricks/cli/libs/dyn" | ||
| ) | ||
|
|
||
| type pipelineTestingEnabled struct{ bundle.RO } | ||
|
|
||
| // PipelineTestingEnabled rejects the testing property of pipelines while the experimental feature is off. | ||
| func PipelineTestingEnabled() bundle.ReadOnlyMutator { | ||
| return &pipelineTestingEnabled{} | ||
| } | ||
|
|
||
| func (v *pipelineTestingEnabled) Name() string { | ||
| return "validate:pipeline_testing_enabled" | ||
| } | ||
|
|
||
| func (v *pipelineTestingEnabled) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { | ||
| if env.ExperimentalPipelineTesting(ctx) { | ||
| return nil | ||
| } | ||
| var diags diag.Diagnostics | ||
| pipelines := b.Config.Resources.Pipelines | ||
| for _, key := range slices.Sorted(maps.Keys(pipelines)) { | ||
| if pipelines[key] == nil || pipelines[key].Testing == nil { | ||
| continue | ||
| } | ||
| path := dyn.NewPath(dyn.Key("resources"), dyn.Key("pipelines"), dyn.Key(key), dyn.Key("testing")) | ||
| value, err := dyn.GetByPath(b.Config.Value(), path) | ||
| if err != nil { | ||
| return diag.FromErr(err) | ||
| } | ||
| diags = append(diags, diag.Diagnostic{ | ||
| Severity: diag.Error, | ||
| Summary: fmt.Sprintf("the testing property of pipelines is experimental. Set %s=true to enable it", env.ExperimentalPipelineTestingVariable), | ||
| Locations: value.Locations(), | ||
| Paths: dyn.ToStructPaths(path), | ||
| }) | ||
| } | ||
| return diags | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| package validate | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/databricks/cli/bundle" | ||
| "github.com/databricks/cli/bundle/config" | ||
| "github.com/databricks/cli/bundle/config/resources" | ||
| "github.com/databricks/cli/bundle/env" | ||
| "github.com/databricks/cli/bundle/internal/bundletest" | ||
| "github.com/databricks/cli/libs/diag" | ||
| libsenv "github.com/databricks/cli/libs/env" | ||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestPipelineTestingEnabled(t *testing.T) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we need this unit test? The acceptance test already shows the error with its location and path (the |
||
| b := &bundle.Bundle{ | ||
| Config: config.Root{ | ||
| Resources: config.Resources{ | ||
| Pipelines: map[string]*resources.Pipeline{ | ||
| "orders": {Testing: &resources.PipelineTesting{}}, | ||
| "reports": {}, | ||
| }, | ||
| }, | ||
| }, | ||
| } | ||
| loc := diag.Location{File: "databricks.yml", Line: 9, Column: 9} | ||
| bundletest.SetLocation(b, "resources.pipelines.orders.testing", []diag.Location{loc}) | ||
|
|
||
| diags := PipelineTestingEnabled().Apply(t.Context(), b) | ||
| require.Len(t, diags, 1) | ||
| assert.Equal(t, diag.Error, diags[0].Severity) | ||
| assert.Contains(t, diags[0].Summary, env.ExperimentalPipelineTestingVariable) | ||
| assert.Equal(t, []diag.Location{loc}, diags[0].Locations) | ||
| require.Len(t, diags[0].Paths, 1) | ||
| assert.Equal(t, "resources.pipelines.orders.testing", diags[0].Paths[0].String()) | ||
|
|
||
| ctx := libsenv.Set(t.Context(), env.ExperimentalPipelineTestingVariable, "true") | ||
| assert.Empty(t, PipelineTestingEnabled().Apply(ctx, b)) | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -80,6 +80,10 @@ var knownMissingInStateType = map[string][]string{ | |
| "alerts": { | ||
| "file_path", | ||
| }, | ||
| "pipelines": { | ||
| // The experimental testing property is not deployed yet. | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If this lands before #7438, a bundle with the variable on and |
||
| "testing", | ||
| }, | ||
| "apps": { | ||
| "lifecycle.prevent_destroy", | ||
| "lifecycle.depends_on", | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| package env | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| envlib "github.com/databricks/cli/libs/env" | ||
| ) | ||
|
|
||
| // ExperimentalPipelineTestingVariable enables the experimental testing property of pipelines. | ||
| const ExperimentalPipelineTestingVariable = "DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING" | ||
|
|
||
| // ExperimentalPipelineTesting reports whether the experimental testing property of pipelines is | ||
| // enabled. While it is off, bundles deploy pipelines exactly as they did before the property existed. | ||
| func ExperimentalPipelineTesting(ctx context.Context) bool { | ||
| enabled, _ := envlib.GetBool(ctx, ExperimentalPipelineTestingVariable) | ||
| return enabled | ||
| } |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
pipeline_development_deprecated.goright above this in initialize.go does the same walk withdyn.MapByPatternonresources.pipelines.*.development, which gives the path and locations directly. Can we do the same here forresources.pipelines.*.testinginstead of iterating the typed map and callingdyn.GetByPathper key?