From 86997dc926e356132a238f0aae504a420281b508 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Wed, 11 Nov 2026 21:56:55 +0000 Subject: [PATCH] Add the testing property of pipelines to bundles Add resources.pipelines..testing, with the test sources and test-only dependencies that databricks pipelines test runs with. The CLI has its own types for it, because the Go SDK does not have the field yet. The property is experimental. It needs DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true. While the variable is off, bundle validate rejects testing and names the variable. Bundles without testing are not changed. Later PRs translate its paths and deploy it. Co-authored-by: Isaac --- .../bundle/refschema/output/pipelines.txt | 8 ++ .../pipelines/testing/databricks.yml | 19 +++++ .../resources/pipelines/testing/out.test.toml | 2 + .../resources/pipelines/testing/output.txt | 33 ++++++++ .../pipelines/testing/pyproject.toml | 6 ++ .../bundle/resources/pipelines/testing/script | 4 + .../resources/pipelines/testing/test.toml | 5 ++ .../pipelines/testing/tests/requirements.txt | 1 + .../pipelines/testing/tests/test_orders.py | 2 + .../testing/transformations/orders.py | 1 + bundle/config/resources/pipeline.go | 28 +++++++ .../validate/pipeline_testing_enabled.go | 49 ++++++++++++ .../validate/pipeline_testing_enabled_test.go | 41 ++++++++++ bundle/direct/dresources/type_test.go | 4 + bundle/env/experimental_pipeline_testing.go | 17 ++++ bundle/internal/schema/annotations.yml | 20 +++++ bundle/phases/initialize.go | 4 + bundle/schema/jsonschema.json | 78 +++++++++++++++++++ .../databricks/bundles/pipelines/__init__.py | 24 ++++++ .../bundles/pipelines/_models/pipeline.py | 18 +++++ .../_models/pipeline_test_environment.py | 42 ++++++++++ .../pipelines/_models/pipeline_test_source.py | 46 +++++++++++ .../pipelines/_models/pipeline_testing.py | 60 ++++++++++++++ 23 files changed, 512 insertions(+) create mode 100644 acceptance/bundle/resources/pipelines/testing/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing/pyproject.toml create mode 100644 acceptance/bundle/resources/pipelines/testing/script create mode 100644 acceptance/bundle/resources/pipelines/testing/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing/tests/requirements.txt create mode 100644 acceptance/bundle/resources/pipelines/testing/tests/test_orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing/transformations/orders.py create mode 100644 bundle/config/validate/pipeline_testing_enabled.go create mode 100644 bundle/config/validate/pipeline_testing_enabled_test.go create mode 100644 bundle/env/experimental_pipeline_testing.go create mode 100644 python/databricks/bundles/pipelines/_models/pipeline_test_environment.py create mode 100644 python/databricks/bundles/pipelines/_models/pipeline_test_source.py create mode 100644 python/databricks/bundles/pipelines/_models/pipeline_testing.py diff --git a/acceptance/bundle/refschema/output/pipelines.txt b/acceptance/bundle/refschema/output/pipelines.txt index c8bc57f512c..cee0902e460 100644 --- a/acceptance/bundle/refschema/output/pipelines.txt +++ b/acceptance/bundle/refschema/output/pipelines.txt @@ -867,6 +867,14 @@ resources.pipelines.*.storage string ALL resources.pipelines.*.tags map[string]string ALL resources.pipelines.*.tags.* string ALL resources.pipelines.*.target string ALL +resources.pipelines.*.testing *resources.PipelineTesting INPUT +resources.pipelines.*.testing.environment *resources.PipelineTestEnvironment INPUT +resources.pipelines.*.testing.environment.dependencies []string INPUT +resources.pipelines.*.testing.environment.dependencies[*] string INPUT +resources.pipelines.*.testing.sources []resources.PipelineTestSource INPUT +resources.pipelines.*.testing.sources[*] resources.PipelineTestSource INPUT +resources.pipelines.*.testing.sources[*].glob *pipelines.PathPattern INPUT +resources.pipelines.*.testing.sources[*].glob.include string INPUT resources.pipelines.*.trigger *pipelines.PipelineTrigger ALL resources.pipelines.*.trigger.cron *pipelines.CronTrigger ALL resources.pipelines.*.trigger.cron.quartz_cron_schedule string ALL diff --git a/acceptance/bundle/resources/pipelines/testing/databricks.yml b/acceptance/bundle/resources/pipelines/testing/databricks.yml new file mode 100644 index 00000000000..346cc9d7963 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: pipeline-testing + +resources: + pipelines: + orders: + name: orders + libraries: + - glob: + include: ./transformations/** + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + - -e .[test] + - -r ./tests/requirements.txt + - ./dist/*.whl diff --git a/acceptance/bundle/resources/pipelines/testing/out.test.toml b/acceptance/bundle/resources/pipelines/testing/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt new file mode 100644 index 00000000000..6133e93eb75 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -0,0 +1,33 @@ + +>>> [CLI] bundle validate -o json +{ + "environment": { + "dependencies": [ + "-e .[test]", + "-r ./tests/requirements.txt", + "./dist/*.whl" + ] + }, + "sources": [ + { + "glob": { + "include": "./tests/**" + } + } + ] +} + +=== Without pipeline testing, the property is an error + +>>> musterr env -u DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING [CLI] bundle validate +Error: the testing property of pipelines is experimental. Set DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true to enable it + at resources.pipelines.orders.testing + in databricks.yml:12:9 + +Name: pipeline-testing +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default + +Found 1 error diff --git a/acceptance/bundle/resources/pipelines/testing/pyproject.toml b/acceptance/bundle/resources/pipelines/testing/pyproject.toml new file mode 100644 index 00000000000..1c198acc7cb --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "orders" +version = "0.1" + +[project.optional-dependencies] +test = ["faker"] diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script new file mode 100644 index 00000000000..1dc320485b7 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -0,0 +1,4 @@ +trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing + +title "Without pipeline testing, the property is an error" +trace musterr env -u DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING $CLI bundle validate diff --git a/acceptance/bundle/resources/pipelines/testing/test.toml b/acceptance/bundle/resources/pipelines/testing/test.toml new file mode 100644 index 00000000000..965c596a302 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/test.toml @@ -0,0 +1,5 @@ +# Runs only against the local test server. +Cloud = false +RecordRequests = true +Ignore = [".databricks"] +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" diff --git a/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt b/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt new file mode 100644 index 00000000000..9d7f6e827a1 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt @@ -0,0 +1 @@ +faker diff --git a/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing/transformations/orders.py new file mode 100644 index 00000000000..3a22621c3bf --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/transformations/orders.py @@ -0,0 +1 @@ +from pyspark import pipelines as dp diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index fd1fc61ce86..1b8b8e0ef43 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -22,6 +22,34 @@ type Pipeline struct { // Nil (unset) uses the server default, which cascades. Set to false to retain datasets // when the pipeline is deleted. Delete-time only: never sent on create/update. CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` + + // Testing is PipelineSpec.testing, which the Go SDK does not have yet. Remove it, and the + // types below, once CreatePipeline has it. + Testing *PipelineTesting `json:"testing,omitempty"` +} + +// PipelineTesting is the test configuration of a pipeline. Only test-only updates use it. +type PipelineTesting struct { + // Sources are the workspace files and folders that hold the pipeline's tests. They must + // not overlap the pipeline's libraries. + Sources []PipelineTestSource `json:"sources,omitempty"` + + // Environment lists the packages that only the tests need. + Environment *PipelineTestEnvironment `json:"environment,omitempty"` +} + +// PipelineTestSource is a workspace location that holds pipeline tests. +type PipelineTestSource struct { + // Glob is a test file, or a folder followed by /**. + Glob *pipelines.PathPattern `json:"glob,omitempty"` +} + +// PipelineTestEnvironment lists the packages that a pipeline's tests need in addition to the +// pipeline's environment. +type PipelineTestEnvironment struct { + // Dependencies are files that declare the packages: a project folder, a requirements + // file, or a wheel. Package names and URLs are rejected by the API. + Dependencies []string `json:"dependencies,omitempty"` } func (p *Pipeline) UnmarshalJSON(b []byte) error { diff --git a/bundle/config/validate/pipeline_testing_enabled.go b/bundle/config/validate/pipeline_testing_enabled.go new file mode 100644 index 00000000000..85fdb85ed4b --- /dev/null +++ b/bundle/config/validate/pipeline_testing_enabled.go @@ -0,0 +1,49 @@ +package validate + +import ( + "context" + "fmt" + "maps" + "slices" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type pipelineTestingEnabled struct{ bundle.RO } + +// PipelineTestingEnabled rejects the testing property of pipelines while the experimental feature is off. +func PipelineTestingEnabled() bundle.ReadOnlyMutator { + return &pipelineTestingEnabled{} +} + +func (v *pipelineTestingEnabled) Name() string { + return "validate:pipeline_testing_enabled" +} + +func (v *pipelineTestingEnabled) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if env.ExperimentalPipelineTesting(ctx) { + return nil + } + var diags diag.Diagnostics + pipelines := b.Config.Resources.Pipelines + for _, key := range slices.Sorted(maps.Keys(pipelines)) { + if pipelines[key] == nil || pipelines[key].Testing == nil { + continue + } + path := dyn.NewPath(dyn.Key("resources"), dyn.Key("pipelines"), dyn.Key(key), dyn.Key("testing")) + value, err := dyn.GetByPath(b.Config.Value(), path) + if err != nil { + return diag.FromErr(err) + } + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("the testing property of pipelines is experimental. Set %s=true to enable it", env.ExperimentalPipelineTestingVariable), + Locations: value.Locations(), + Paths: dyn.ToStructPaths(path), + }) + } + return diags +} diff --git a/bundle/config/validate/pipeline_testing_enabled_test.go b/bundle/config/validate/pipeline_testing_enabled_test.go new file mode 100644 index 00000000000..e8acc03b9a6 --- /dev/null +++ b/bundle/config/validate/pipeline_testing_enabled_test.go @@ -0,0 +1,41 @@ +package validate + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/diag" + libsenv "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPipelineTestingEnabled(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "orders": {Testing: &resources.PipelineTesting{}}, + "reports": {}, + }, + }, + }, + } + loc := diag.Location{File: "databricks.yml", Line: 9, Column: 9} + bundletest.SetLocation(b, "resources.pipelines.orders.testing", []diag.Location{loc}) + + diags := PipelineTestingEnabled().Apply(t.Context(), b) + require.Len(t, diags, 1) + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Contains(t, diags[0].Summary, env.ExperimentalPipelineTestingVariable) + assert.Equal(t, []diag.Location{loc}, diags[0].Locations) + require.Len(t, diags[0].Paths, 1) + assert.Equal(t, "resources.pipelines.orders.testing", diags[0].Paths[0].String()) + + ctx := libsenv.Set(t.Context(), env.ExperimentalPipelineTestingVariable, "true") + assert.Empty(t, PipelineTestingEnabled().Apply(ctx, b)) +} diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index d7515ae55a8..bf64ef100c4 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -80,6 +80,10 @@ var knownMissingInStateType = map[string][]string{ "alerts": { "file_path", }, + "pipelines": { + // The experimental testing property is not deployed yet. + "testing", + }, "apps": { "lifecycle.prevent_destroy", "lifecycle.depends_on", diff --git a/bundle/env/experimental_pipeline_testing.go b/bundle/env/experimental_pipeline_testing.go new file mode 100644 index 00000000000..b00a94176e9 --- /dev/null +++ b/bundle/env/experimental_pipeline_testing.go @@ -0,0 +1,17 @@ +package env + +import ( + "context" + + envlib "github.com/databricks/cli/libs/env" +) + +// ExperimentalPipelineTestingVariable enables the experimental testing property of pipelines. +const ExperimentalPipelineTestingVariable = "DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING" + +// ExperimentalPipelineTesting reports whether the experimental testing property of pipelines is +// enabled. While it is off, bundles deploy pipelines exactly as they did before the property existed. +func ExperimentalPipelineTesting(ctx context.Context) bool { + enabled, _ := envlib.GetBool(ctx, ExperimentalPipelineTestingVariable) + return enabled +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 230a7459dd7..c98179a445c 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1665,6 +1665,26 @@ resources: "user_name": "description": |- The name of the user granted the permission level. + "testing": + "description": |- + The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. It is experimental and needs `DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true`. + "x-databricks-launch-stage": |- + PRIVATE_PREVIEW + "$fields": + "environment": + "description": |- + The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not. + "$fields": + "dependencies": + "description": |- + A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `-e` and an extra, for example `-e ./project[test]`. A relative path, such as `-e ./project[test]` or `-r ./tests/requirements.txt`, is relative to the YAML file that defines it. Package names and URLs are rejected. + "sources": + "description": |- + The workspace files and folders that hold the pipeline's tests. + "$fields": + "glob": + "description": |- + A `.py` test file, or a folder followed by `/**`. Other wildcards are rejected, and the sources must not overlap the pipeline's `libraries`. A relative path is relative to the YAML file that defines it. "trigger": "deprecation_message": |- Use continuous instead diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 8c5a68d3130..b4a08596bb1 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -166,6 +166,10 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Must run after PythonMutator to see development values set by Python mutators. validate.PipelineDevelopmentDeprecated(), + // Reject the experimental testing property of pipelines while it is off. Must run after + // PythonMutator, which can set testing, and before TranslatePaths and the deploy use it. + validate.PipelineTestingEnabled(), + // Compute resources.volumes.*.volume_path and resolve references to it. Must run after // PythonMutator: volume_path is computed and read-only, not declared by the PyDABs Volume // model, so exposing it to Python would fail resource loading (like "deployment" below). diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index afb5a80db72..f18aa38d5b3 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2444,6 +2444,12 @@ "deprecationMessage": "This field is deprecated", "deprecated": true }, + "testing": { + "description": "[Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. It is experimental and needs `DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true`.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PipelineTesting", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, "trigger": { "description": "Which pipeline trigger to use. Deprecated: Use `continuous` instead.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger", @@ -2500,6 +2506,64 @@ } ] }, + "resources.PipelineTestEnvironment": { + "oneOf": [ + { + "type": "object", + "properties": { + "dependencies": { + "description": "A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `-e` and an extra, for example `-e ./project[test]`. A relative path, such as `-e ./project[test]` or `-r ./tests/requirements.txt`, is relative to the YAML file that defines it. Package names and URLs are rejected.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.PipelineTestSource": { + "oneOf": [ + { + "type": "object", + "properties": { + "glob": { + "description": "A `.py` test file, or a folder followed by `/**`. Other wildcards are rejected, and the sources must not overlap the pipeline's `libraries`. A relative path is relative to the YAML file that defines it.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PathPattern" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.PipelineTesting": { + "oneOf": [ + { + "type": "object", + "properties": { + "environment": { + "description": "The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PipelineTestEnvironment" + }, + "sources": { + "description": "The workspace files and folders that hold the pipeline's tests.", + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.PipelineTestSource" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.PostgresBranch": { "oneOf": [ { @@ -22808,6 +22872,20 @@ } ] }, + "resources.PipelineTestSource": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PipelineTestSource" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.SecretScopePermission": { "oneOf": [ { diff --git a/python/databricks/bundles/pipelines/__init__.py b/python/databricks/bundles/pipelines/__init__.py index 412cbfa59b3..d9bf369bdb3 100644 --- a/python/databricks/bundles/pipelines/__init__.py +++ b/python/databricks/bundles/pipelines/__init__.py @@ -207,6 +207,15 @@ "PipelinePermissionLevel", "PipelinePermissionLevelParam", "PipelinePermissionParam", + "PipelineTestEnvironment", + "PipelineTestEnvironmentDict", + "PipelineTestEnvironmentParam", + "PipelineTestSource", + "PipelineTestSourceDict", + "PipelineTestSourceParam", + "PipelineTesting", + "PipelineTestingDict", + "PipelineTestingParam", "PipelinesEnvironment", "PipelinesEnvironmentDict", "PipelinesEnvironmentParam", @@ -645,6 +654,21 @@ PipelinePermissionLevel, PipelinePermissionLevelParam, ) +from databricks.bundles.pipelines._models.pipeline_test_environment import ( + PipelineTestEnvironment, + PipelineTestEnvironmentDict, + PipelineTestEnvironmentParam, +) +from databricks.bundles.pipelines._models.pipeline_test_source import ( + PipelineTestSource, + PipelineTestSourceDict, + PipelineTestSourceParam, +) +from databricks.bundles.pipelines._models.pipeline_testing import ( + PipelineTesting, + PipelineTestingDict, + PipelineTestingParam, +) from databricks.bundles.pipelines._models.pipelines_environment import ( PipelinesEnvironment, PipelinesEnvironmentDict, diff --git a/python/databricks/bundles/pipelines/_models/pipeline.py b/python/databricks/bundles/pipelines/_models/pipeline.py index d9d898db40f..7ae0298551d 100644 --- a/python/databricks/bundles/pipelines/_models/pipeline.py +++ b/python/databricks/bundles/pipelines/_models/pipeline.py @@ -41,6 +41,10 @@ PipelinePermission, PipelinePermissionParam, ) +from databricks.bundles.pipelines._models.pipeline_testing import ( + PipelineTesting, + PipelineTestingParam, +) from databricks.bundles.pipelines._models.pipelines_environment import ( PipelinesEnvironment, PipelinesEnvironmentParam, @@ -254,6 +258,13 @@ class Pipeline(Resource): [DEPRECATED] Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field. """ + testing: VariableOrOptional[PipelineTesting] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. It is experimental and needs `DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true`. + """ + usage_policy_id: VariableOrOptional[str] = None """ :meta private: [EXPERIMENTAL] @@ -467,6 +478,13 @@ class PipelineDict(TypedDict, total=False): [DEPRECATED] Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field. """ + testing: VariableOrOptional[PipelineTestingParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. It is experimental and needs `DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true`. + """ + usage_policy_id: VariableOrOptional[str] """ :meta private: [EXPERIMENTAL] diff --git a/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py b/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py new file mode 100644 index 00000000000..172f9878502 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py @@ -0,0 +1,42 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PipelineTestEnvironment: + """ + :meta private: [EXPERIMENTAL] + """ + + dependencies: VariableOrList[str] = field(default_factory=list) + """ + A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `-e` and an extra, for example `-e ./project[test]`. A relative path, such as `-e ./project[test]` or `-r ./tests/requirements.txt`, is relative to the YAML file that defines it. Package names and URLs are rejected. + """ + + @classmethod + def from_dict(cls, value: "PipelineTestEnvironmentDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PipelineTestEnvironmentDict": + return _transform_to_json_value(self) # type:ignore + + +class PipelineTestEnvironmentDict(TypedDict, total=False): + """""" + + dependencies: VariableOrList[str] + """ + A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `-e` and an extra, for example `-e ./project[test]`. A relative path, such as `-e ./project[test]` or `-r ./tests/requirements.txt`, is relative to the YAML file that defines it. Package names and URLs are rejected. + """ + + +PipelineTestEnvironmentParam = PipelineTestEnvironmentDict | PipelineTestEnvironment diff --git a/python/databricks/bundles/pipelines/_models/pipeline_test_source.py b/python/databricks/bundles/pipelines/_models/pipeline_test_source.py new file mode 100644 index 00000000000..3aefd8ca585 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/pipeline_test_source.py @@ -0,0 +1,46 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional +from databricks.bundles.pipelines._models.path_pattern import ( + PathPattern, + PathPatternParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PipelineTestSource: + """ + :meta private: [EXPERIMENTAL] + """ + + glob: VariableOrOptional[PathPattern] = None + """ + A `.py` test file, or a folder followed by `/**`. Other wildcards are rejected, and the sources must not overlap the pipeline's `libraries`. A relative path is relative to the YAML file that defines it. + """ + + @classmethod + def from_dict(cls, value: "PipelineTestSourceDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PipelineTestSourceDict": + return _transform_to_json_value(self) # type:ignore + + +class PipelineTestSourceDict(TypedDict, total=False): + """""" + + glob: VariableOrOptional[PathPatternParam] + """ + A `.py` test file, or a folder followed by `/**`. Other wildcards are rejected, and the sources must not overlap the pipeline's `libraries`. A relative path is relative to the YAML file that defines it. + """ + + +PipelineTestSourceParam = PipelineTestSourceDict | PipelineTestSource diff --git a/python/databricks/bundles/pipelines/_models/pipeline_testing.py b/python/databricks/bundles/pipelines/_models/pipeline_testing.py new file mode 100644 index 00000000000..6a027157f02 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/pipeline_testing.py @@ -0,0 +1,60 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional +from databricks.bundles.pipelines._models.pipeline_test_environment import ( + PipelineTestEnvironment, + PipelineTestEnvironmentParam, +) +from databricks.bundles.pipelines._models.pipeline_test_source import ( + PipelineTestSource, + PipelineTestSourceParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PipelineTesting: + """ + :meta private: [EXPERIMENTAL] + """ + + environment: VariableOrOptional[PipelineTestEnvironment] = None + """ + The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not. + """ + + sources: VariableOrList[PipelineTestSource] = field(default_factory=list) + """ + The workspace files and folders that hold the pipeline's tests. + """ + + @classmethod + def from_dict(cls, value: "PipelineTestingDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PipelineTestingDict": + return _transform_to_json_value(self) # type:ignore + + +class PipelineTestingDict(TypedDict, total=False): + """""" + + environment: VariableOrOptional[PipelineTestEnvironmentParam] + """ + The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not. + """ + + sources: VariableOrList[PipelineTestSourceParam] + """ + The workspace files and folders that hold the pipeline's tests. + """ + + +PipelineTestingParam = PipelineTestingDict | PipelineTesting