diff --git a/acceptance/bundle/refschema/output/pipelines.txt b/acceptance/bundle/refschema/output/pipelines.txt index c8bc57f512c..cee0902e460 100644 --- a/acceptance/bundle/refschema/output/pipelines.txt +++ b/acceptance/bundle/refschema/output/pipelines.txt @@ -867,6 +867,14 @@ resources.pipelines.*.storage string ALL resources.pipelines.*.tags map[string]string ALL resources.pipelines.*.tags.* string ALL resources.pipelines.*.target string ALL +resources.pipelines.*.testing *resources.PipelineTesting INPUT +resources.pipelines.*.testing.environment *resources.PipelineTestEnvironment INPUT +resources.pipelines.*.testing.environment.dependencies []string INPUT +resources.pipelines.*.testing.environment.dependencies[*] string INPUT +resources.pipelines.*.testing.sources []resources.PipelineTestSource INPUT +resources.pipelines.*.testing.sources[*] resources.PipelineTestSource INPUT +resources.pipelines.*.testing.sources[*].glob *pipelines.PathPattern INPUT +resources.pipelines.*.testing.sources[*].glob.include string INPUT resources.pipelines.*.trigger *pipelines.PipelineTrigger ALL resources.pipelines.*.trigger.cron *pipelines.CronTrigger ALL resources.pipelines.*.trigger.cron.quartz_cron_schedule string ALL diff --git a/acceptance/bundle/resources/pipelines/testing/databricks.yml b/acceptance/bundle/resources/pipelines/testing/databricks.yml new file mode 100644 index 00000000000..346cc9d7963 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: pipeline-testing + +resources: + pipelines: + orders: + name: orders + libraries: + - glob: + include: ./transformations/** + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + - -e .[test] + - -r ./tests/requirements.txt + - ./dist/*.whl diff --git a/acceptance/bundle/resources/pipelines/testing/out.test.toml b/acceptance/bundle/resources/pipelines/testing/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt new file mode 100644 index 00000000000..6133e93eb75 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -0,0 +1,33 @@ + +>>> [CLI] bundle validate -o json +{ + "environment": { + "dependencies": [ + "-e .[test]", + "-r ./tests/requirements.txt", + "./dist/*.whl" + ] + }, + "sources": [ + { + "glob": { + "include": "./tests/**" + } + } + ] +} + +=== Without pipeline testing, the property is an error + +>>> musterr env -u DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING [CLI] bundle validate +Error: the testing property of pipelines is experimental. Set DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true to enable it + at resources.pipelines.orders.testing + in databricks.yml:12:9 + +Name: pipeline-testing +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default + +Found 1 error diff --git a/acceptance/bundle/resources/pipelines/testing/pyproject.toml b/acceptance/bundle/resources/pipelines/testing/pyproject.toml new file mode 100644 index 00000000000..1c198acc7cb --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "orders" +version = "0.1" + +[project.optional-dependencies] +test = ["faker"] diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script new file mode 100644 index 00000000000..1dc320485b7 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -0,0 +1,4 @@ +trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing + +title "Without pipeline testing, the property is an error" +trace musterr env -u DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING $CLI bundle validate diff --git a/acceptance/bundle/resources/pipelines/testing/test.toml b/acceptance/bundle/resources/pipelines/testing/test.toml new file mode 100644 index 00000000000..965c596a302 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/test.toml @@ -0,0 +1,5 @@ +# Runs only against the local test server. +Cloud = false +RecordRequests = true +Ignore = [".databricks"] +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" diff --git a/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt b/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt new file mode 100644 index 00000000000..9d7f6e827a1 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt @@ -0,0 +1 @@ +faker diff --git a/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing/transformations/orders.py new file mode 100644 index 00000000000..3a22621c3bf --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/transformations/orders.py @@ -0,0 +1 @@ +from pyspark import pipelines as dp diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index fd1fc61ce86..1b8b8e0ef43 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -22,6 +22,34 @@ type Pipeline struct { // Nil (unset) uses the server default, which cascades. Set to false to retain datasets // when the pipeline is deleted. Delete-time only: never sent on create/update. CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` + + // Testing is PipelineSpec.testing, which the Go SDK does not have yet. Remove it, and the + // types below, once CreatePipeline has it. + Testing *PipelineTesting `json:"testing,omitempty"` +} + +// PipelineTesting is the test configuration of a pipeline. Only test-only updates use it. +type PipelineTesting struct { + // Sources are the workspace files and folders that hold the pipeline's tests. They must + // not overlap the pipeline's libraries. + Sources []PipelineTestSource `json:"sources,omitempty"` + + // Environment lists the packages that only the tests need. + Environment *PipelineTestEnvironment `json:"environment,omitempty"` +} + +// PipelineTestSource is a workspace location that holds pipeline tests. +type PipelineTestSource struct { + // Glob is a test file, or a folder followed by /**. + Glob *pipelines.PathPattern `json:"glob,omitempty"` +} + +// PipelineTestEnvironment lists the packages that a pipeline's tests need in addition to the +// pipeline's environment. +type PipelineTestEnvironment struct { + // Dependencies are files that declare the packages: a project folder, a requirements + // file, or a wheel. Package names and URLs are rejected by the API. + Dependencies []string `json:"dependencies,omitempty"` } func (p *Pipeline) UnmarshalJSON(b []byte) error { diff --git a/bundle/config/validate/pipeline_testing_enabled.go b/bundle/config/validate/pipeline_testing_enabled.go new file mode 100644 index 00000000000..85fdb85ed4b --- /dev/null +++ b/bundle/config/validate/pipeline_testing_enabled.go @@ -0,0 +1,49 @@ +package validate + +import ( + "context" + "fmt" + "maps" + "slices" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type pipelineTestingEnabled struct{ bundle.RO } + +// PipelineTestingEnabled rejects the testing property of pipelines while the experimental feature is off. +func PipelineTestingEnabled() bundle.ReadOnlyMutator { + return &pipelineTestingEnabled{} +} + +func (v *pipelineTestingEnabled) Name() string { + return "validate:pipeline_testing_enabled" +} + +func (v *pipelineTestingEnabled) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if env.ExperimentalPipelineTesting(ctx) { + return nil + } + var diags diag.Diagnostics + pipelines := b.Config.Resources.Pipelines + for _, key := range slices.Sorted(maps.Keys(pipelines)) { + if pipelines[key] == nil || pipelines[key].Testing == nil { + continue + } + path := dyn.NewPath(dyn.Key("resources"), dyn.Key("pipelines"), dyn.Key(key), dyn.Key("testing")) + value, err := dyn.GetByPath(b.Config.Value(), path) + if err != nil { + return diag.FromErr(err) + } + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("the testing property of pipelines is experimental. Set %s=true to enable it", env.ExperimentalPipelineTestingVariable), + Locations: value.Locations(), + Paths: dyn.ToStructPaths(path), + }) + } + return diags +} diff --git a/bundle/config/validate/pipeline_testing_enabled_test.go b/bundle/config/validate/pipeline_testing_enabled_test.go new file mode 100644 index 00000000000..e8acc03b9a6 --- /dev/null +++ b/bundle/config/validate/pipeline_testing_enabled_test.go @@ -0,0 +1,41 @@ +package validate + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/diag" + libsenv "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPipelineTestingEnabled(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "orders": {Testing: &resources.PipelineTesting{}}, + "reports": {}, + }, + }, + }, + } + loc := diag.Location{File: "databricks.yml", Line: 9, Column: 9} + bundletest.SetLocation(b, "resources.pipelines.orders.testing", []diag.Location{loc}) + + diags := PipelineTestingEnabled().Apply(t.Context(), b) + require.Len(t, diags, 1) + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Contains(t, diags[0].Summary, env.ExperimentalPipelineTestingVariable) + assert.Equal(t, []diag.Location{loc}, diags[0].Locations) + require.Len(t, diags[0].Paths, 1) + assert.Equal(t, "resources.pipelines.orders.testing", diags[0].Paths[0].String()) + + ctx := libsenv.Set(t.Context(), env.ExperimentalPipelineTestingVariable, "true") + assert.Empty(t, PipelineTestingEnabled().Apply(ctx, b)) +} diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index d7515ae55a8..bf64ef100c4 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -80,6 +80,10 @@ var knownMissingInStateType = map[string][]string{ "alerts": { "file_path", }, + "pipelines": { + // The experimental testing property is not deployed yet. + "testing", + }, "apps": { "lifecycle.prevent_destroy", "lifecycle.depends_on", diff --git a/bundle/env/experimental_pipeline_testing.go b/bundle/env/experimental_pipeline_testing.go new file mode 100644 index 00000000000..b00a94176e9 --- /dev/null +++ b/bundle/env/experimental_pipeline_testing.go @@ -0,0 +1,17 @@ +package env + +import ( + "context" + + envlib "github.com/databricks/cli/libs/env" +) + +// ExperimentalPipelineTestingVariable enables the experimental testing property of pipelines. +const ExperimentalPipelineTestingVariable = "DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING" + +// ExperimentalPipelineTesting reports whether the experimental testing property of pipelines is +// enabled. While it is off, bundles deploy pipelines exactly as they did before the property existed. +func ExperimentalPipelineTesting(ctx context.Context) bool { + enabled, _ := envlib.GetBool(ctx, ExperimentalPipelineTestingVariable) + return enabled +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 230a7459dd7..c98179a445c 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1665,6 +1665,26 @@ resources: "user_name": "description": |- The name of the user granted the permission level. + "testing": + "description": |- + The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. It is experimental and needs `DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true`. + "x-databricks-launch-stage": |- + PRIVATE_PREVIEW + "$fields": + "environment": + "description": |- + The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not. + "$fields": + "dependencies": + "description": |- + A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `-e` and an extra, for example `-e ./project[test]`. A relative path, such as `-e ./project[test]` or `-r ./tests/requirements.txt`, is relative to the YAML file that defines it. Package names and URLs are rejected. + "sources": + "description": |- + The workspace files and folders that hold the pipeline's tests. + "$fields": + "glob": + "description": |- + A `.py` test file, or a folder followed by `/**`. Other wildcards are rejected, and the sources must not overlap the pipeline's `libraries`. A relative path is relative to the YAML file that defines it. "trigger": "deprecation_message": |- Use continuous instead diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 8c5a68d3130..b4a08596bb1 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -166,6 +166,10 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Must run after PythonMutator to see development values set by Python mutators. validate.PipelineDevelopmentDeprecated(), + // Reject the experimental testing property of pipelines while it is off. Must run after + // PythonMutator, which can set testing, and before TranslatePaths and the deploy use it. + validate.PipelineTestingEnabled(), + // Compute resources.volumes.*.volume_path and resolve references to it. Must run after // PythonMutator: volume_path is computed and read-only, not declared by the PyDABs Volume // model, so exposing it to Python would fail resource loading (like "deployment" below). diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index afb5a80db72..f18aa38d5b3 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2444,6 +2444,12 @@ "deprecationMessage": "This field is deprecated", "deprecated": true }, + "testing": { + "description": "[Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. It is experimental and needs `DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true`.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PipelineTesting", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, "trigger": { "description": "Which pipeline trigger to use. Deprecated: Use `continuous` instead.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger", @@ -2500,6 +2506,64 @@ } ] }, + "resources.PipelineTestEnvironment": { + "oneOf": [ + { + "type": "object", + "properties": { + "dependencies": { + "description": "A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `-e` and an extra, for example `-e ./project[test]`. A relative path, such as `-e ./project[test]` or `-r ./tests/requirements.txt`, is relative to the YAML file that defines it. Package names and URLs are rejected.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.PipelineTestSource": { + "oneOf": [ + { + "type": "object", + "properties": { + "glob": { + "description": "A `.py` test file, or a folder followed by `/**`. Other wildcards are rejected, and the sources must not overlap the pipeline's `libraries`. A relative path is relative to the YAML file that defines it.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PathPattern" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.PipelineTesting": { + "oneOf": [ + { + "type": "object", + "properties": { + "environment": { + "description": "The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PipelineTestEnvironment" + }, + "sources": { + "description": "The workspace files and folders that hold the pipeline's tests.", + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.PipelineTestSource" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.PostgresBranch": { "oneOf": [ { @@ -22808,6 +22872,20 @@ } ] }, + "resources.PipelineTestSource": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PipelineTestSource" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.SecretScopePermission": { "oneOf": [ { diff --git a/python/databricks/bundles/pipelines/__init__.py b/python/databricks/bundles/pipelines/__init__.py index 412cbfa59b3..d9bf369bdb3 100644 --- a/python/databricks/bundles/pipelines/__init__.py +++ b/python/databricks/bundles/pipelines/__init__.py @@ -207,6 +207,15 @@ "PipelinePermissionLevel", "PipelinePermissionLevelParam", "PipelinePermissionParam", + "PipelineTestEnvironment", + "PipelineTestEnvironmentDict", + "PipelineTestEnvironmentParam", + "PipelineTestSource", + "PipelineTestSourceDict", + "PipelineTestSourceParam", + "PipelineTesting", + "PipelineTestingDict", + "PipelineTestingParam", "PipelinesEnvironment", "PipelinesEnvironmentDict", "PipelinesEnvironmentParam", @@ -645,6 +654,21 @@ PipelinePermissionLevel, PipelinePermissionLevelParam, ) +from databricks.bundles.pipelines._models.pipeline_test_environment import ( + PipelineTestEnvironment, + PipelineTestEnvironmentDict, + PipelineTestEnvironmentParam, +) +from databricks.bundles.pipelines._models.pipeline_test_source import ( + PipelineTestSource, + PipelineTestSourceDict, + PipelineTestSourceParam, +) +from databricks.bundles.pipelines._models.pipeline_testing import ( + PipelineTesting, + PipelineTestingDict, + PipelineTestingParam, +) from databricks.bundles.pipelines._models.pipelines_environment import ( PipelinesEnvironment, PipelinesEnvironmentDict, diff --git a/python/databricks/bundles/pipelines/_models/pipeline.py b/python/databricks/bundles/pipelines/_models/pipeline.py index d9d898db40f..7ae0298551d 100644 --- a/python/databricks/bundles/pipelines/_models/pipeline.py +++ b/python/databricks/bundles/pipelines/_models/pipeline.py @@ -41,6 +41,10 @@ PipelinePermission, PipelinePermissionParam, ) +from databricks.bundles.pipelines._models.pipeline_testing import ( + PipelineTesting, + PipelineTestingParam, +) from databricks.bundles.pipelines._models.pipelines_environment import ( PipelinesEnvironment, PipelinesEnvironmentParam, @@ -254,6 +258,13 @@ class Pipeline(Resource): [DEPRECATED] Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field. """ + testing: VariableOrOptional[PipelineTesting] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. It is experimental and needs `DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true`. + """ + usage_policy_id: VariableOrOptional[str] = None """ :meta private: [EXPERIMENTAL] @@ -467,6 +478,13 @@ class PipelineDict(TypedDict, total=False): [DEPRECATED] Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field. """ + testing: VariableOrOptional[PipelineTestingParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. It is experimental and needs `DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true`. + """ + usage_policy_id: VariableOrOptional[str] """ :meta private: [EXPERIMENTAL] diff --git a/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py b/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py new file mode 100644 index 00000000000..172f9878502 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py @@ -0,0 +1,42 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PipelineTestEnvironment: + """ + :meta private: [EXPERIMENTAL] + """ + + dependencies: VariableOrList[str] = field(default_factory=list) + """ + A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `-e` and an extra, for example `-e ./project[test]`. A relative path, such as `-e ./project[test]` or `-r ./tests/requirements.txt`, is relative to the YAML file that defines it. Package names and URLs are rejected. + """ + + @classmethod + def from_dict(cls, value: "PipelineTestEnvironmentDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PipelineTestEnvironmentDict": + return _transform_to_json_value(self) # type:ignore + + +class PipelineTestEnvironmentDict(TypedDict, total=False): + """""" + + dependencies: VariableOrList[str] + """ + A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `-e` and an extra, for example `-e ./project[test]`. A relative path, such as `-e ./project[test]` or `-r ./tests/requirements.txt`, is relative to the YAML file that defines it. Package names and URLs are rejected. + """ + + +PipelineTestEnvironmentParam = PipelineTestEnvironmentDict | PipelineTestEnvironment diff --git a/python/databricks/bundles/pipelines/_models/pipeline_test_source.py b/python/databricks/bundles/pipelines/_models/pipeline_test_source.py new file mode 100644 index 00000000000..3aefd8ca585 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/pipeline_test_source.py @@ -0,0 +1,46 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional +from databricks.bundles.pipelines._models.path_pattern import ( + PathPattern, + PathPatternParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PipelineTestSource: + """ + :meta private: [EXPERIMENTAL] + """ + + glob: VariableOrOptional[PathPattern] = None + """ + A `.py` test file, or a folder followed by `/**`. Other wildcards are rejected, and the sources must not overlap the pipeline's `libraries`. A relative path is relative to the YAML file that defines it. + """ + + @classmethod + def from_dict(cls, value: "PipelineTestSourceDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PipelineTestSourceDict": + return _transform_to_json_value(self) # type:ignore + + +class PipelineTestSourceDict(TypedDict, total=False): + """""" + + glob: VariableOrOptional[PathPatternParam] + """ + A `.py` test file, or a folder followed by `/**`. Other wildcards are rejected, and the sources must not overlap the pipeline's `libraries`. A relative path is relative to the YAML file that defines it. + """ + + +PipelineTestSourceParam = PipelineTestSourceDict | PipelineTestSource diff --git a/python/databricks/bundles/pipelines/_models/pipeline_testing.py b/python/databricks/bundles/pipelines/_models/pipeline_testing.py new file mode 100644 index 00000000000..6a027157f02 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/pipeline_testing.py @@ -0,0 +1,60 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional +from databricks.bundles.pipelines._models.pipeline_test_environment import ( + PipelineTestEnvironment, + PipelineTestEnvironmentParam, +) +from databricks.bundles.pipelines._models.pipeline_test_source import ( + PipelineTestSource, + PipelineTestSourceParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PipelineTesting: + """ + :meta private: [EXPERIMENTAL] + """ + + environment: VariableOrOptional[PipelineTestEnvironment] = None + """ + The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not. + """ + + sources: VariableOrList[PipelineTestSource] = field(default_factory=list) + """ + The workspace files and folders that hold the pipeline's tests. + """ + + @classmethod + def from_dict(cls, value: "PipelineTestingDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PipelineTestingDict": + return _transform_to_json_value(self) # type:ignore + + +class PipelineTestingDict(TypedDict, total=False): + """""" + + environment: VariableOrOptional[PipelineTestEnvironmentParam] + """ + The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not. + """ + + sources: VariableOrList[PipelineTestSourceParam] + """ + The workspace files and folders that hold the pipeline's tests. + """ + + +PipelineTestingParam = PipelineTestingDict | PipelineTesting